From e5e588dab5422aae71f99a7401a27e401132dd8c Mon Sep 17 00:00:00 2001 From: TaykYoku Date: Sun, 14 Feb 2021 04:45:34 +0100 Subject: [PATCH 001/170] merge auth PR to intagration(v7r2-pre33) --- docs/source/dirac.cfg | 717 ++++++++++++++ .../API/ConfigurationHandler.py | 73 ++ src/DIRAC/ConfigurationSystem/API/__init__.py | 5 + .../Client/Helpers/Registry.py | 386 +++++--- .../Client/Helpers/Resources.py | 95 +- .../ConfigurationSystem/Client/PathFinder.py | 2 + .../ConfigurationSystem/Client/Utilities.py | 6 +- .../test/Test_agentOptions.py | 2 +- src/DIRAC/Core/Base/API.py | 9 +- src/DIRAC/Core/Base/DB.py | 44 +- src/DIRAC/Core/DISET/AuthManager.py | 448 +++++---- src/DIRAC/Core/DISET/RequestHandler.py | 11 +- src/DIRAC/Core/DISET/ThreadConfig.py | 27 +- src/DIRAC/Core/DISET/private/BaseClient.py | 22 +- src/DIRAC/Core/DISET/test/Test_AuthManager.py | 54 +- src/DIRAC/Core/Security/Locations.py | 14 +- src/DIRAC/Core/Security/VOMSService.py | 9 +- .../Core/Tornado/Server/BaseRequestHandler.py | 704 +++++++++++++ .../Core/Tornado/Server/HandlerManager.py | 262 +++-- src/DIRAC/Core/Tornado/Server/TornadoREST.py | 85 ++ .../Core/Tornado/Server/TornadoServer.py | 329 ++++++- .../Core/Tornado/Server/TornadoService.py | 642 +----------- src/DIRAC/Core/Tornado/Web/Conf.py | 426 ++++++++ src/DIRAC/Core/Tornado/Web/CoreHandler.py | 35 + src/DIRAC/Core/Tornado/Web/HandlerMgr.py | 189 ++++ src/DIRAC/Core/Tornado/Web/SessionData.py | 184 ++++ src/DIRAC/Core/Tornado/Web/StaticHandler.py | 30 + src/DIRAC/Core/Tornado/Web/TemplateLoader.py | 32 + src/DIRAC/Core/Tornado/Web/WebHandler.py | 379 +++++++ src/DIRAC/Core/Tornado/Web/__init__.py | 3 + .../scripts/tornado_start_endpoints.py | 61 ++ .../Core/Tornado/scripts/tornado_start_web.py | 71 ++ src/DIRAC/Core/Utilities/DErrno.py | 3 + src/DIRAC/Core/Utilities/DictCache.py | 18 + src/DIRAC/Core/Utilities/Proxy.py | 67 +- src/DIRAC/Core/Utilities/Shifter.py | 21 +- src/DIRAC/Core/scripts/dirac_webapp_run.py | 53 + .../DataManagementSystem/Agent/FTS3Agent.py | 11 +- .../Service/FTS3ManagerHandler.py | 6 +- .../Service/FileCatalogProxyHandler.py | 4 +- .../Service/StorageElementProxyHandler.py | 2 +- src/DIRAC/FrameworkSystem/API/AuthHandler.py | 403 ++++++++ src/DIRAC/FrameworkSystem/API/ProxyHandler.py | 87 ++ src/DIRAC/FrameworkSystem/API/__init__.py | 5 + .../Client/AuthManagerClient.py | 134 +++ .../FrameworkSystem/Client/AuthManagerData.py | 176 ++++ .../FrameworkSystem/Client/ProxyGeneration.py | 17 +- .../Client/ProxyManagerClient.py | 521 ++++------ .../Client/ProxyManagerData.py | 213 ++++ src/DIRAC/FrameworkSystem/ConfigTemplate.cfg | 11 + src/DIRAC/FrameworkSystem/DB/AuthDB2.py | 262 +++++ src/DIRAC/FrameworkSystem/DB/AuthDB2.sql | 2 + src/DIRAC/FrameworkSystem/DB/ProxyDB.py | 921 +++++++++--------- .../Service/AuthManagerHandler.py | 387 ++++++++ .../Service/ProxyManagerHandler.py | 751 ++++++++++---- src/DIRAC/FrameworkSystem/Utilities/halo.py | 683 +++++++++++++ .../private/authorization/AuthServer.py | 286 ++++++ .../private/authorization/__init__.py | 9 + .../authorization/grants/AuthorizationCode.py | 140 +++ .../authorization/grants/DeviceFlow.py | 81 ++ .../authorization/grants/ImplicitFlow.py | 58 ++ .../authorization/grants/RefreshToken.py | 56 ++ .../private/authorization/grants/__init__.py | 14 + .../private/authorization/utils/Clients.py | 94 ++ .../private/authorization/utils/Requests.py | 47 + .../private/authorization/utils/Sessions.py | 180 ++++ .../private/authorization/utils/Tokens.py | 115 +++ .../private/authorization/utils/__init__.py | 14 + .../private/testNotebookAuth.py | 131 +++ .../scripts/dirac_admin_get_proxy.py | 60 +- .../scripts/dirac_admin_users_with_proxy.py | 41 +- .../scripts/dirac_notebook_proxy_init.py | 161 +++ .../scripts/dirac_proxy_destroy.py | 26 +- .../scripts/dirac_proxy_get_uploaded_info.py | 27 +- .../scripts/dirac_proxy_info.py | 19 +- .../scripts/dirac_proxy_init.py | 283 +++++- src/DIRAC/Interfaces/API/DiracAdmin.py | 535 ++++++---- .../RequestManagementSystem/Client/Request.py | 17 +- .../private/OperationHandlerBase.py | 34 +- .../private/RequestTask.py | 24 +- .../private/RequestValidator.py | 8 +- .../Resources/Catalog/FileCatalogClient.py | 10 +- .../Resources/Catalog/LcgFileCatalogClient.py | 4 +- .../Computing/GlobusComputingElement.py | 12 +- src/DIRAC/Resources/IdProvider/IdProvider.py | 60 +- .../Resources/IdProvider/IdProviderFactory.py | 58 +- .../Resources/IdProvider/OAuth2IdProvider.py | 271 ++++++ .../ProxyProvider/DIRACCAProxyProvider.py | 17 +- .../ProxyProvider/OAuth2ProxyProvider.py | 167 ++++ .../ProxyProvider/PUSPProxyProvider.py | 5 +- .../Resources/ProxyProvider/ProxyProvider.py | 38 +- .../ProxyProvider/ProxyProviderFactory.py | 10 +- .../test/Test_DIRACCAProxyProvider.py | 29 +- .../test/Test_ProxyProviderFactory.py | 43 +- .../Agent/TaskManagerAgentBase.py | 7 +- .../Client/TaskManager.py | 18 +- .../Agent/JobAgent.py | 15 +- .../Agent/SiteDirector.py | 26 +- .../Agent/test/Test_Agent_JobAgent.py | 10 +- .../Client/Matcher.py | 8 +- .../ConfigTemplate.cfg | 2 + .../DB/PilotAgentsDB.py | 10 +- .../Service/JobManagerHandler.py | 9 +- .../Service/JobMonitoringHandler.py | 3 +- .../Service/JobPolicy.py | 70 +- .../Service/PilotManagerHandler.py | 13 +- .../Service/WMSUtilities.py | 25 +- .../private/ConfigHelper.py | 45 +- .../correctors/BaseHistoryCorrector.py | 4 +- test | 0 test.sh | 2 + tests/Integration/Framework/Test_ProxyDB.py | 153 ++- .../FIXME_IntegrationFCT.py | 104 +- .../Test_Client_Req.py | 2 +- .../Test_DIRACCAProxyProvider.py | 20 +- .../Test_PilotsClient.py | 26 +- 116 files changed, 10916 insertions(+), 2954 deletions(-) create mode 100644 docs/source/dirac.cfg create mode 100644 src/DIRAC/ConfigurationSystem/API/ConfigurationHandler.py create mode 100644 src/DIRAC/ConfigurationSystem/API/__init__.py create mode 100644 src/DIRAC/Core/Tornado/Server/BaseRequestHandler.py create mode 100644 src/DIRAC/Core/Tornado/Server/TornadoREST.py create mode 100644 src/DIRAC/Core/Tornado/Web/Conf.py create mode 100644 src/DIRAC/Core/Tornado/Web/CoreHandler.py create mode 100644 src/DIRAC/Core/Tornado/Web/HandlerMgr.py create mode 100644 src/DIRAC/Core/Tornado/Web/SessionData.py create mode 100644 src/DIRAC/Core/Tornado/Web/StaticHandler.py create mode 100644 src/DIRAC/Core/Tornado/Web/TemplateLoader.py create mode 100644 src/DIRAC/Core/Tornado/Web/WebHandler.py create mode 100644 src/DIRAC/Core/Tornado/Web/__init__.py create mode 100644 src/DIRAC/Core/Tornado/scripts/tornado_start_endpoints.py create mode 100644 src/DIRAC/Core/Tornado/scripts/tornado_start_web.py create mode 100644 src/DIRAC/Core/scripts/dirac_webapp_run.py create mode 100644 src/DIRAC/FrameworkSystem/API/AuthHandler.py create mode 100644 src/DIRAC/FrameworkSystem/API/ProxyHandler.py create mode 100644 src/DIRAC/FrameworkSystem/API/__init__.py create mode 100644 src/DIRAC/FrameworkSystem/Client/AuthManagerClient.py create mode 100644 src/DIRAC/FrameworkSystem/Client/AuthManagerData.py create mode 100644 src/DIRAC/FrameworkSystem/Client/ProxyManagerData.py create mode 100644 src/DIRAC/FrameworkSystem/DB/AuthDB2.py create mode 100644 src/DIRAC/FrameworkSystem/DB/AuthDB2.sql create mode 100644 src/DIRAC/FrameworkSystem/Service/AuthManagerHandler.py create mode 100644 src/DIRAC/FrameworkSystem/Utilities/halo.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/AuthServer.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/__init__.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/grants/AuthorizationCode.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/grants/DeviceFlow.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/grants/ImplicitFlow.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/grants/RefreshToken.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/grants/__init__.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/utils/Clients.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/utils/Requests.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/utils/Sessions.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/utils/Tokens.py create mode 100644 src/DIRAC/FrameworkSystem/private/authorization/utils/__init__.py create mode 100644 src/DIRAC/FrameworkSystem/private/testNotebookAuth.py create mode 100644 src/DIRAC/FrameworkSystem/scripts/dirac_notebook_proxy_init.py create mode 100644 src/DIRAC/Resources/IdProvider/OAuth2IdProvider.py create mode 100644 src/DIRAC/Resources/ProxyProvider/OAuth2ProxyProvider.py create mode 100644 test create mode 100755 test.sh diff --git a/docs/source/dirac.cfg b/docs/source/dirac.cfg new file mode 100644 index 00000000000..48f2f2a05d5 --- /dev/null +++ b/docs/source/dirac.cfg @@ -0,0 +1,717 @@ +### Registry section: +# Sections to register VOs, groups, users and hosts +# https://dirac.readthedocs.org/en/latest/AdministratorGuide/UserManagement.html +Registry +{ + + ## Registry options: + # Default user group to be used: + DefaultGroup = lhcb_user + + # Querantine user group is usually to be used in case you want to set + # users in groups by hand as a "punishment" for a certain period of time: + QuarantineGroup = lowPriority_user + + # Default proxy time expressed in seconds: + DefaultProxyTime = 4000 + ## + + # Trusted hosts section, subsections represents host name of the DIRAC secondary servers + Hosts + { + + dirac.host.com + { + + # Host distinguish name obtained from host certificate + DN = /O=MyOrg/OU=Unity/CN=dirac.host.com + + # Properties associated with the host + Properties = JobAdministrator + Properties += FullDelegation + Properties += Operator + Properties += CSAdministrator + Properties += ProductionManagement + Properties += AlarmsManagement + Properties += ProxyManagement + Properties += TrustedHost + } + } + + ## VOs: + # DIRAC VOs section, subsections represents name of the DIRAC VO or alias name of the real VOMS VO + VO + { + + # It is not mandatory for the DIRAC VO to have the same name as the corresponding VOMS VO + lhcb + { + + # VO administrator user name, that also MUST be registered(/Registry/Users section) + VOAdmin = lhcbadmin + + # VO administrator group used for querying VOMS server. + # If not specified, the VO "DefaultGroup" will be used + VOAdminGroup = lhcb_admin + + # Real VOMS VO name, if this VO is associated with VOMS VO + VOMSName = lhcb + + # Section to describe all the VOMS servers that can be used with the given VOMS VO + VOMSServers + { + + # The host name of the VOMS server + cclcgvomsli01.in2p3.fr + { + + # DN of the VOMS server certificate + DN = /O=GRID-FR/C=FR/O=CNRS/OU=CC-IN2P3/CN=cclcgvomsli01.in2p3.fr + + # The VOMS server port + Port = 15003 + + # CA that issued the VOMS server certificate + CA = /C=FR/O=CNRS/CN=GRID2-FR + } + } + } + } + ## + + ## Groups: + # DIRAC groups section, subsections represents the name of the group + Groups + { + + # Group for the common user + lhcb_user + { + + # DIRAC users logins than belongs to the group + Users = lhcbuser1 + + # Group properties(set permissions of the group users) + Properties = NormalUser # Normal user operations + + # Permission to download proxy with this group, by default: True + DownloadableProxy = False + + # Role of the users in the VO + VOMSRole = /lhcb + + # Virtual organization associated with the group + VOMSVO = lhcb + + # Just for normal users: + JobShare = 200 + + # Controls automatic Proxy upload by dirac-proxy-init: + AutoUploadProxy = True + + # Controls automatic Proxy upload by dirac-proxy-init for Pilot groups: + AutoUploadPilotProxy = True + + # Controls automatic addition of VOMS extension by dirac-proxy-init: + AutoAddVOMS = True + } + + # Group to submit pilot jobs + lhcb_pilot + { + Properties = GenericPilot # Generic pilot + Properties += LimitedDelegation # Allow getting only limited proxies (ie. pilots) + Properties += Pilot # Private pilot + } + + # Admin group + lhcb_admin + { + Properties = AlarmsManagement # Allow to set notifications and manage alarms + Properties += ServiceAdministrator # DIRAC Service Administrator + Properties += CSAdministrator # possibility to edit the Configuration Service + Properties += JobAdministrator # Job Administrator can manipulate everybody's jobs + Properties += FullDelegation # Allow getting full delegated proxies + Properties += ProxyManagement # Allow managing proxies + Properties += Operator # Operator + } + } + ## + + ## Users: + # DIRAC users section, subsections represents the name of the user + Users + { + + lhcbuser1 + { + # Distinguish name obtained from user certificate (Mandatory) + DN = /O=My organisation/C=FR/OU=Unit/CN=My Name + + # User e-mail (Mandatory) + Email = my@email.com + + # Cellular phone number + mobile = +030621555555 + + # Quota assigned to the user. Expressed in MBs. + Quota = 300 + + # This subsection describes the properties associated with each DN attribute (optional) + DNProperties + { + + # Arbitrary section name + DNSubsection + { + + # Distinguish name obtained from user certificate (Mandatory) + DN = /O=My organisation/C=FR/OU=Unit/CN=My Name + + # Proxy provider that can generate the proxy certificate with DN in DN attribute. + ProxyProviders = MY_DIRACCA + } + } + } + } + ## +} +### + +Systems +{ +# the systems section is automatically obtained from the ConfigTemplate.cfg files and can be found at +# https://dirac.readthedocs.org/en/latest/AdministratorGuide/Configuration/ExampleConfig.html + DataManagementSystem + { + Agents + { + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts3.html#fts3agent + FTS3Agent + { + OperationBulkSize = 20 # How many Operation we will treat in one loop + JobBulkSize = 20 # How many Job we will monitor in one loop + MaxFilesPerJob = 100 # Max number of files to go in a single job + MaxAttemptsPerFile = 256 # Max number of attempt per file + DeleteGraceDays = 180 # days before removing jobs + DeleteLimitPerCycle = 100 # Max number of deletes per cycle + KickAssignedHours = 1 # hours before kicking jobs with old assignment tag + KickLimitPerCycle = 100 # Max number of kicks per cycle + + } + } + Services + { + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/dfc.html#filecataloghandler + FileCatalogHandler + { + Port = 9197 + DatasetManager = DatasetManager + DefaultUmask = 0775 + DirectoryManager = DirectoryLevelTree + DirectoryMetadata = DirectoryMetadata + FileManager = FileManager + FileMetadata = FileMetadata + GlobalReadAccess = True + LFNPFNConvention = Strong + ResolvePFN = True + SecurityManager = NoSecurityManager + SEManager = SEManagerDB + UniqueGUID = False + UserGroupManager = UserAndGroupManagerDB + ValidFileStatus = [AprioriGoodTrashRemovingProbing] + ValidReplicaStatus = [AprioriGoodTrashRemovingProbing] + VisibleFileStatus = [AprioriGood] + VisibleReplicaStatus = [AprioriGood] + + } + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.html#ftsmanager + FTS3ManagerHandler + { + # No specific configuration + Port = 9193 + } + } + Databases + { + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/dfc.html#filecatalogdb + FileCatalogDB + { + # No specific configuration + DBName = FileCatalogDB + } + FTS3DB + { + # No specific configuration + DBName = FTS3DB + } + } + } + RequestManagementSystem + { + Agents + { + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#cleanreqdbagent + CleanReqDBAgent + { + DeleteGraceDays = 60 # Delay after which Requests are removed + DeleteLimit = 100 # Maximum number of Requests to remove per cycle + DeleteFailed = False # Whether to delete also Failed request + KickGraceHours = 1 # After how long we should kick the Requests in `Assigned` + KickLimit = 10000 # Maximum number of requests kicked by cycle + } + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#requestexecutingagent + RequestExecutingAgent + { + BulkRequest = 0 + MinProcess = 1 + MaxProcess = 8 + ProcessPoolQueueSize = 25 + ProcessPoolTimeout = 900 + ProcessTaskTimeout = 900 + ProcessPoolSleep = 4 + RequestsPerCycle = 50 + + # Define the different Operation types + # see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsObjects.html#operation-types + OperationHandlers + { + DummyOperation + { + # These parameters can be defined for all handlers + + # The location of the python file, without .py, is mandatory + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/DummyHandler # Mandatory + LogLevel = DEBUG # self explanatory + MaxAttemts = 256 # Maximum attempts per file + TimeOut = 300 # Timeout in seconds of the operation + TimeOutPerFile = 40 # Additional delay per file + } + ForwardDISET{ + Location = DIRAC/RequestManagementSystem/Agent/RequestOperations/ForwardDISET + } + MoveReplica + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica + } + PutAndRegister + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister + } + RegisterFile + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile + } + RegisterReplica + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica + } + RemoveFile + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile + } + RemoveReplica + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica + } + ReplicateAndRegister + { + Location = DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister + FTSMode = True # If True, will use FTS to transfer files + FTSBannedGroups = lhcb_user # list of groups for which not to use FTS + } + SetFileStatus + { + Location = DIRAC/TransformationSystem/Agent/RequestOperations/SetFileStatus + } + } + } + } + Databases + { + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#requestdb + RequestDB + { + # No specific configuration + DBName = RequestDB + } + } + Services + { + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#reqmanager + ReqManager + { + Port = 9140 + constantRequestDelay = 0 # Constant delay when retrying a request + } + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#reqproxy + ReqProxy + { + Port = 9161 + } + } + URLs + { + # Yes.... it is ReqProxyURLs, and not ReqProxy... + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/RequestManagement/rmsComponents.html#reqproxy + ReqProxyURLs = dips://server1:9161/RequestManagement/ReqProxy, dips://server2:9161/RequestManagement/ReqProxy + } + } + TransformationSystem + { + Agents + { + ##BEGIN TransformationCleaningAgent + TransformationCleaningAgent + { + # MetaData key to use to identify output data + TransfIDMeta=TransformationID + + # Location of the OutputData, if the OutputDirectories parameter is not set for + # transformations only 'MetadataCatalog has to be used + DirectoryLocations=TransformationDB,MetadataCatalog + + # Enable or disable, default enabled + EnableFlag=True + + # How many days to wait before archiving transformations + ArchiveAfter=7 + + # Shifter to use for removal operations, default is empty and + # using the transformation owner for cleanup + shifterProxy= + + # Which transformation types to clean + # If not filled, transformation types are taken from + # Operations/Transformations/DataManipulation + # and Operations/Transformations/DataProcessing + TransformationTypes= + + #Time between cycles in seconds + PollingTime = 3600 + } + ##END + } + } + Framework + { + Services + { + ComponentMonitoring + { + Port = 9190 + # This enables ES monitoring only for this particular service. + EnableActivityMonitoring = yes + Authorization + { + Default = ServiceAdministrator + componentExists = authenticated + getComponents = authenticated + hostExists = authenticated + getHosts = authenticated + installationExists = authenticated + getInstallations = authenticated + updateLog = Operator + } + } + } + } +} +Resources +{ + + # Section for proxy providers, subsections is the names of the proxy providers + # https://dirac.readthedocs.org/en/latest/AdministratorGuide/Resources/proxyprovider.html + ProxyProviders + { + + ## DIRACCA type: + MY_DIRACCA + { + + # Main option, to show which proxy provider type you want to register. + ProviderType = DIRACCA + + # The path to the CA certificate. This option is required. + CertFile = /opt/dirac/etc/grid-security/DIRACCA-EOSH/cert.pem + + # The path to the CA key. This option is required. + KeyFile = /opt/dirac/etc/grid-security/DIRACCA-EOSH/key.pem + + # The distinguished name fields that must contain the exact same contents as that field in the CA's + # DN. If this parameter is not specified, the default value will be a empty list. + Match = O, OU + + # The distinguished name fields list that must be present. If this parameter is not specified, the + # default value will be a "CN". + Supplied = C, CN + + # The distinguished name fields list that are allowed, but not required. If this parameter is not + # specified, the default value will be a "C, O, OU, emailAddress" + Optional = emailAddress + + # Order of the distinguished name fields in a created user certificate. If this parameter is not + # specified, the default value will be a "C, O, OU, CN, emailAddress" + DNOrder = C, O, OU, emailAddress, CN + + # To set default value for distinguished name field. + C = FR + O = DIRAC + OU = DIRAC TEST + + # The path to the openssl configuration file. This is optional and not recomended to use. + # But if you choose to use this option, it is recommended to use a relatively simple configuration. + # All required parameters will be taken from the configuration file, except "DNOrder". + CAConfigFile = /opt/dirac/pro/etc/openssl_config_ca.cnf + } + ## + + ## PUSP type: + MY_PUSP + { + + ProviderType = DIRACCA + + # PUSP service URL + ServiceURL = https://mypuspserver.com/ + } + ## + + ## OAuth2 type: + MY_OAuth2 + { + + ProviderType = OAuth2 + + # Authorization server's issuer identifier URL + issuer = https://masterportal-pilot.aai.egi.eu/mp-oa2-server + + # Identifier of OAuth client + client_id = myproxy:oa4mp,2012:/client_id/aca7c8dfh439fewjb298fdb + + # Secret key of OAuth client + client_secret = ISh-Q32bkXRf-HD2hdh93d-hd20DH2-wqedwiU@S22 + + # OAuth2 parameter specified in https://tools.ietf.org/html/rfc6749 + prompt = consent + + # Some specific parameter for specific proxy provider + max_proxylifetime = 864000 + proxy_endpoint = https://masterportal-pilot.aai.egi.eu/mp-oa2-server/getproxy + } + ## + } + + #Where all your Catalogs are defined + FileCatalogs + { + #There is one section per catalog + #See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.html + + { + CatalogType = # used for plugin selection + CatalogURL = # used for DISET URL + } + } + #FTS endpoint definition http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.htmlfts-servers-definition + # Passed to the constructor of the pluginFTSEndpoints + { + FTS3 + { + CERN-FTS3 = https://fts3.cern.ch:8446 + } + } + #Abstract definition of storage elements, used to be inherited. + #see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storage/index.htmlstorageelementbases + StorageElementBases + { + #The base SE definition can contain all the options of a normal SE + #http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storage/index.htmlstorageelements + CERN-EOS + { + BackendType = eos # backend type of storage element + SEType = T0D1 # Tape or Disk SE + UseCatalogURL = True # used the stored url or generate it (default False) + ReadAccess = True # Allowed for Read if no RSS enabled + WriteAccess = True # Allowed for Write if no RSS enabled + CheckAccess = True # Allowed for Check if no RSS enabled + RemoveAccess = True # Allowed for Remove if no RSS enabled + #Protocol section, see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storage/index.htmlavailable-protocol-plugins + GFAL2_SRM2 + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + PluginName = GFAL2_SRM2 # If different from the section name + Protocol = srm # primary protocol + Path = /eos/lhcb/grid/prod # base path + Access = remote + SpaceToken = LHCb-EOS + WSUrl = /srm/v2/server?SFN= + } + } + } + #http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storage/index.htmlstorageelements + StorageElements + { + #Just inherit everything from CERN-EOS, without change + CERN-DST-EOS + { + BaseSE = CERN-EOS + } + CERN-USER # inherit from CERN-EOS + { + BaseSE = CERN-EOS + #Modify the options for Gfal2 + GFAL2_SRM2 + { + Path = /eos/lhcb/grid/user + SpaceToken = LHCb_USER + } + } + # Abstract definition of storage elements, used to be inherited. + # see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storages/index.html#storageelementbases + StorageElementBases + { + # The base SE definition can contain all the options of a normal SE + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storages/index.html#storageelements + CERN-EOS + { + BackendType = eos # backend type of storage element + SEType = T0D1 # Tape or Disk SE + UseCatalogURL = True # used the stored url or generate it (default False) + ReadAccess = True # Allowed for Read if no RSS enabled + WriteAccess = True # Allowed for Write if no RSS enabled + CheckAccess = True # Allowed for Check if no RSS enabled + RemoveAccess = True # Allowed for Remove if no RSS enabled + OccupancyLFN = /lhcb/storageDetails.json # Json containing occupancy details + SpaceReservation = LHCb-EOS # Space reservation name if any. Concept like SpaceToken + # Protocol section, see # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storages/index.html#available-protocol-plugins + GFAL2_SRM2 + { + Host = srm-eoslhcb.cern.ch + Port = 8443 + PluginName = GFAL2_SRM2 # If different from the section name + Protocol = srm # primary protocol + Path = /eos/lhcb/grid/prod # base path + Access = remote + SpaceToken = LHCb-EOS + WSUrl = /srm/v2/server?SFN= + } + } + } + # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storages/index.html#storageelements + StorageElements + { + CERN-DST-EOS # Just inherit everything from CERN-EOS, without change + { + BaseSE = CERN-EOS + } + CERN-USER # inherit from CERN-EOS + { + BaseSE = CERN-EOS + GFAL2_SRM2 # Modify the options for Gfal2 + { + Path = /eos/lhcb/grid/user + SpaceToken = LHCb_USER + } + GFAL2_XROOT # Add an extra protocol + { + Host = eoslhcb.cern.ch + Port = 8443 + Protocol = root + Path = /eos/lhcb/grid/user + Access = remote + SpaceToken = LHCb-EOS + WSUrl = /srm/v2/server?SFN= + } + } + CERN-ALIAS + { + Alias = CERN-USER # Use CERN-USER when instanciating CERN-ALIAS + } + } + # See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storages/index.html#storageelementgroups + StorageElementGroups + { + CERN-Storages = CERN-DST-EOS, CERN-USER + } + +} +Operations +{ + #This is the default section of operations. + #Any value here can be overwriten in the setup specific section + Defaults + { + # This will globally enable ES based monitoring for Service and AgentModule. + EnableActivityMonitoring = yes + DataManagement + { + #see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.html#multi-protocol + #for the next 4 options + AccessProtocols = srm + AccessProtocols += dips + RegistrationProtocols = srm + RegistrationProtocols += dips + # + StageProtocols = srm + ThirdPartyProtocols = srm + WriteProtocols = srm + WriteProtocols += dips + #FTS related options. See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.html + FTSVersion = FTS3 # should only be that... + FTSPlacement + { + FTS3 + { + ServerPolicy = Random # http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.html#ftsserver-policy + } + } + } + # Options for the pilot3 + # See https://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/WorkloadManagement/Pilots/Pilots3.html + Pilot + { + pilotRepo = https://github.com/DIRACGrid/Pilot.git # git repository of the pilot + pilotScriptsPath = Pilot # Path to the code, inside the Git repository | + pilotRepoBranch = master # Branch to use + pilotVORepo = https://github.com/MyDIRAC/VOPilot.git # git repository of the pilot extension + pilotVOScriptsPath = VOPilot # Path to the code, inside the Git repository + pilotVORepoBranch = master # Branch to use + uploadToWebApp = True # Try to upload the files from the CS to the list of servers + workDir = /tmp/pilot3Files # Local work directory on the masterCS for synchronisation + } + Services + { + #See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.html + Catalogs + { + CatalogList = Catalog1 + CatalogList += Catalog2 + CatalogList += etc # List of catalogs defined in Resources to use + #Each catalog defined in Resources should also contain some runtime options here + + { + Status = Active # enable the catalog or not (default Active) + AccessType = Read-Write # No default + AccessType += must be set + Master = True # See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.html#master-catalog + #Dynamic conditions to enable or not the catalog + #See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Catalog/index.htmlconditional-filecatalogs + Conditions + { + WRITE = + READ = + ALL = + = + } + } + } + } + } + #Options in this section will only be used when running with the + # setup + + { + } +} diff --git a/src/DIRAC/ConfigurationSystem/API/ConfigurationHandler.py b/src/DIRAC/ConfigurationSystem/API/ConfigurationHandler.py new file mode 100644 index 00000000000..bd61e151379 --- /dev/null +++ b/src/DIRAC/ConfigurationSystem/API/ConfigurationHandler.py @@ -0,0 +1,73 @@ +""" HTTP API of the DIRAC configuration data, rewrite from the RESTDIRAC project +""" +import re +import json + +from tornado import web, gen +from tornado.template import Template + +from DIRAC import S_OK, S_ERROR, gConfig, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers import Resources, Registry +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager + +from DIRAC.Core.Tornado.Server.TornadoREST import TornadoREST + +__RCSID__ = "$Id$" + + +class ConfigurationHandler(TornadoREST): + AUTH_PROPS = "all" + LOCATION = "/DIRAC" + METHOD_PREFIX = 'web_' + + path_conf = ['([a-z]+)'] + def web_conf(self, key): + """ REST endpoint for configuration system: + + **GET** /conf/? -- get configuration information + + Options: + * *path* -- path in the configuration structure, by default it's "/". + * *version* -- the configuration version of the requester, if *version* is newer + than the one present on the server, an empty result will be returned + + Response: + +-----------+---------------------------------------+------------------------+ + | *key* | Description | Type | + +-----------+---------------------------------------+------------------------+ + | dump | Current CFG() | encoded in json format | + +-----------+---------------------------------------+------------------------+ + | option | Option value | text | + +-----------+---------------------------------------+------------------------+ + | options | Options list in a section | encoded in json format | + +-----------+---------------------------------------+------------------------+ + | dict | Options with values in a section | encoded in json format | + +-----------+---------------------------------------+------------------------+ + | sections | Sections list in a section | text | + +-----------+---------------------------------------+------------------------+ + """ + self.log.notice('Request configuration information') + + path = self.get_argument('path', '/') + + version = self.get_argument('version', None) + if version and (version or '0') >= gConfigurationData.getVersion(): + return '' + if key == 'dump': + return str(gConfigurationData.getRemoteCFG()) + elif key == 'option': + return gConfig.getOption(path) + elif key == 'dict': + return gConfig.getOptionsDict(path) + elif key == 'options': + return gConfig.getOptions(path) + elif key == 'sections': + return gConfig.getSections(path) + elif key == 'getGroupsStatusByUsername': + return gProxyManager.getgetGroupsStatusByUsername(**self.get_arguments) + elif any([key == m and re.match('^[a-z][A-z]+', m) for m in dir(Registry)]) and self.isRegisteredUser(): + method = getattr(Registry, key) + return method(**self.get_arguments) + else: + return S_ERROR('%s request unsuported' % key) diff --git a/src/DIRAC/ConfigurationSystem/API/__init__.py b/src/DIRAC/ConfigurationSystem/API/__init__.py new file mode 100644 index 00000000000..2492dd6ec85 --- /dev/null +++ b/src/DIRAC/ConfigurationSystem/API/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +# $HeadURL$ +__RCSID__ = "$Id$" diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py index e9377b1915a..369ec191355 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py @@ -6,18 +6,32 @@ import six import errno +from pprint import pprint from DIRAC import S_OK, S_ERROR +from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO -__RCSID__ = "$Id$" +# Registry use cached data from AuthManager and ProxyManager services +from DIRAC.FrameworkSystem.Client.ProxyManagerData import gProxyManagerData +from DIRAC.FrameworkSystem.Client.AuthManagerData import gAuthManagerData -# pylint: disable=missing-docstring +__RCSID__ = "$Id$" gBaseRegistrySection = "/Registry" +def getVOMSInfo(vo=None, dn=None): + """ Get cached information from VOMS API + + :param list dn: requested DN + + :return: S_OK(dict)/S_ERROR() + """ + return gProxyManagerData.getActualVOMSesDNs(voList=[vo] if vo else vo, dnList=[dn] if dn else dn) + + def getUsernameForDN(dn, usersList=None): """ Find DIRAC user for DN @@ -35,18 +49,30 @@ def getUsernameForDN(dn, usersList=None): for username in usersList: if dn in gConfig.getValue("%s/Users/%s/DN" % (gBaseRegistrySection, username), []): return S_OK(username) - return S_ERROR("No username found for dn %s" % dn) + # Get users profiles from session manager cache + result = gAuthManagerData.getIDsForDN(dn) + if result['OK']: + for uid in result['Value']: + result = getUsernameForID(uid) + if result['OK']: + return result + + return S_ERROR("No username found for dn %s" % dn) +@deprecated("Use getDNsForUsername or getDNsForUsernameFromCS if need information only from CS") def getDNForUsername(username): - """ Get user DN for user + dnList = getDNsForUsernameFromSC(username) + return S_OK(dnList) if dnList else S_ERROR("No DN found for user %s" % username) - :param str username: user name +def getDNsForUsernameFromCS(username): + """ Find DNs for DIRAC user from CS - :return: S_OK(str)/S_ERROR() + :param str username: DIRAC user + + :return: list -- contain DNs """ - dnList = gConfig.getValue("%s/Users/%s/DN" % (gBaseRegistrySection, username), []) - return S_OK(dnList) if dnList else S_ERROR("No DN found for user %s" % username) + return gConfig.getValue("%s/Users/%s/DN" % (gBaseRegistrySection, username), []) def getDNForHost(host): @@ -60,18 +86,53 @@ def getDNForHost(host): return S_OK(dnList) if dnList else S_ERROR("No DN found for host %s" % host) -def getGroupsForDN(dn): +def getGroupsForDN(dn, groupsList=None): """ Get all possible groups for DN :param str dn: user DN + :param list groupsList: group list where need to search :return: S_OK(list)/S_ERROR() -- contain list of groups """ dn = dn.strip() + groups = [] + if not groupsList: + result = gConfig.getSections("%s/Groups" % gBaseRegistrySection) + if not result['OK']: + return result + groupsList = result['Value'] + result = getUsernameForDN(dn) if not result['OK']: return result - return getGroupsForUser(result['Value']) + user = result['Value'] + + # Get VOMS information cache + result = getVOMSInfo(dn=dn) + if not result['OK']: + return result + vomsData = result['Value'] + + result = getVOsWithVOMS() + if not result['OK']: + return result + vomsVOs = result['Value'] + + for group in groupsList: + if user in getGroupOption(group, 'Users', []): + vo = getGroupOption(group, 'VO') + # Is VOMS VO? + if vo in vomsVOs and vomsData.get(vo) and vomsData[vo]['OK'] and vomsData[vo]['Value']: + voData = vomsData[vo]['Value'] + role = getGroupOption(group, 'VOMSRole') + if not role or role in voData[dn]['VOMSRoles']: + groups.append(group) + else: + # If it's not VOMS VO or cannot get information from VOMS + groups.append(group) + groups = list(set(groups)) + groups.sort() + return S_OK(groups) if groups else S_ERROR('No groups found for %s' % dn) def __getGroupsWithAttr(attrName, value): @@ -94,14 +155,27 @@ def __getGroupsWithAttr(attrName, value): return S_OK(groups) if groups else S_ERROR("No groups found for %s=%s" % (attrName, value)) -def getGroupsForUser(username): - """ Find groups for user +def getGroupsForUser(username, groupsList=None): + """ Find groups for user or if set reseachedGroup check it for user :param str username: user name + :param list groupsList: groups - :return: S_OK(list)/S_ERROR() -- contain list of groups + :return: S_OK(list or bool)/S_ERROR() -- contain list of groups or status group for user """ - return __getGroupsWithAttr('Users', username) + if not groupsList: + retVal = gConfig.getSections("%s/Groups" % gBaseRegistrySection) + if not retVal['OK']: + return retVal + groupsList = retVal['Value'] + + groups = [] + for group in groupsList: + if username in getGroupOption(group, 'Users', []): + groups.append(group) + groups = list(set(groups)) + groups.sort() + return S_OK(groups) if groups else S_ERROR('No groups found for %s user' % username) def getGroupsForVO(vo): @@ -205,7 +279,7 @@ def getAllGroups(): return result['Value'] if result['OK'] else [] -def getUsersInGroup(groupName, defaultValue=None): +def getUsersInGroup(group, defaultValue=None): """ Find all users for group :param str group: group name @@ -213,8 +287,9 @@ def getUsersInGroup(groupName, defaultValue=None): :return: list """ - option = "%s/Groups/%s/Users" % (gBaseRegistrySection, groupName) - return gConfig.getValue(option, [] if defaultValue is None else defaultValue) + users = list(set(getGroupOption(group, 'Users', []))) + users.sort() + return users or [] if defaultValue is None else defaultValue def getUsersInVO(vo, defaultValue=None): @@ -225,45 +300,56 @@ def getUsersInVO(vo, defaultValue=None): :return: list """ + users = [] result = getGroupsForVO(vo) - if not result['OK'] or not result['Value']: - return [] if defaultValue is None else defaultValue - groups = result['Value'] + if result['OK'] and result['Value']: + for group in result['Value']: + users += getUsersInGroup(group) + users = list(set(users)) + users.sort() + return users or [] if defaultValue is None else defaultValue - userList = [] - for group in groups: - userList += getUsersInGroup(group) - return userList +def getDNsInGroup(group, checkStatus=False): + """ Find user DNs for DIRAC group -def getDNsInVO(vo): - """ Get all DNs that have a VO users - - :param str vo: VO name + :param str group: group name + :param bool checkStatus: don't add suspended DNs :return: list """ - DNs = [] - for user in getUsersInVO(vo): - result = getDNForUsername(user) - if result['OK']: - DNs.extend(result['Value']) - return DNs - - -def getDNsInGroup(groupName): - """ Find all DNs for DIRAC group + vomsData = {} + vo = getGroupOption(group, 'VO') - :param str groupName: group name + # Get VOMS information for VO, if it's VOMS VO + result = getVOsWithVOMS([vo]) + if not result['OK']: + return result + if result['Value']: + result = getVOMSInfo(vo=vo) + if not result['OK']: + return result + vomsData = result['Value'] - :return: list - """ DNs = [] - for user in getUsersInGroup(groupName): - result = getDNForUsername(user) - if result['OK']: - DNs.extend(result['Value']) - return DNs + for username in getGroupOption(group, 'Users', []): + if checkStatus and vo in getUserOption(username, 'Suspended', []): + continue + result = getDNsForUsername(username) + if not result['OK']: + return result + userDNs = result['Value'] + if vomsData.get(vo) and vomsData[vo]['OK']: + voData = vomsData[vo]['Value'] + role = getGroupOption(group, 'VOMSRole') + for dn in userDNs: + if dn in voData and (not checkStatus or not voData[dn]['Suspended']): + if not role or role in voData[dn]['ActiveRoles' if checkStatus else 'VOMSRoles']: + DNs.append(dn) + else: + DNs += userDNs + + return list(set(DNs)) def getPropertiesForGroup(groupName, defaultValue=None): @@ -300,8 +386,6 @@ def getPropertiesForEntity(group, name="", dn="", defaultValue=None): :return: defaultValue or list """ - if defaultValue is None: - defaultValue = [] if group == 'hosts': if not name: result = getHostnameForDN(dn) @@ -468,15 +552,16 @@ def getVOMSVOForGroup(group): return vomsVO -def getGroupsWithVOMSAttribute(vomsAttr): +def getGroupsWithVOMSAttribute(vomsAttr, groupsList=None): """ Search groups with VOMS attribute :param str vomsAttr: VOMS attribute + :param list groupsList: groups where need to search :return: list """ groups = [] - for group in gConfig.getSections("%s/Groups" % (gBaseRegistrySection)).get('Value', []): + for group in groupsList or getAllGroups(): if vomsAttr == gConfig.getValue("%s/Groups/%s/VOMSRole" % (gBaseRegistrySection, group), ""): groups.append(group) return groups @@ -587,72 +672,30 @@ def getUsernameForID(ID, usersList=None): return S_ERROR("No username found for ID %s" % ID) -def getCAForUsername(username): - """ Get CA option by user name - - :param str username: user name - - :return: S_OK(str)/S_ERROR() - """ - dnList = gConfig.getValue("%s/Users/%s/CA" % (gBaseRegistrySection, username), []) - return S_OK(dnList) if dnList else S_ERROR("No CA found for user %s" % username) - - -def getDNProperty(userDN, value, defaultValue=None): - """ Get property from DNProperties section by user DN +def getDNProperty(dn, prop, defaultValue=None, username=None): + """ Get user DN property - :param str userDN: user DN - :param str value: option that need to get + :param str dn: user DN + :param str prop: property name :param defaultValue: default value + :param str username: username - :return: S_OK()/S_ERROR() -- str or list that contain option value - """ - result = getUsernameForDN(userDN) - if not result['OK']: - return result - pathDNProperties = "%s/Users/%s/DNProperties" % (gBaseRegistrySection, result['Value']) - result = gConfig.getSections(pathDNProperties) - if result['OK']: - for section in result['Value']: - if userDN == gConfig.getValue("%s/%s/DN" % (pathDNProperties, section)): - return S_OK(gConfig.getValue("%s/%s/%s" % (pathDNProperties, section, value), defaultValue)) - return S_OK(defaultValue) - - -def getProxyProvidersForDN(userDN): - """ Get proxy providers by user DN - - :param str userDN: user DN - - :return: S_OK(list)/S_ERROR() + :return: S_OK()/S_ERROR() """ - return getDNProperty(userDN, 'ProxyProviders', []) - - -def getDNFromProxyProviderForUserID(proxyProvider, userID): - """ Get groups by user DN in DNProperties - - :param str proxyProvider: proxy provider name - :param str userID: user identificator + if not username: + result = getUsernameForDN(dn) + if not result['OK']: + return result + username = result['Value'] - :return: S_OK(str)/S_ERROR() - """ - # Get user name - result = getUsernameForID(userID) + root = "%s/Users/%s/DNProperties" % (gBaseRegistrySection, username) + result = gConfig.getSections(root) if not result['OK']: return result - # Get DNs from user - result = getDNForUsername(result['Value']) - if not result['OK']: - return result - for DN in result['Value']: - result = getProxyProvidersForDN(DN) - if not result['OK']: - return result - if proxyProvider in result['Value']: - return S_OK(DN) - return S_ERROR(errno.ENODATA, - "No DN found for %s proxy provider for user ID %s" % (proxyProvider, userID)) + for section in result['Value']: + if dn == gConfig.getValue("%s/%s/DN" % (root, section)): + return S_OK(gConfig.getValue("%s/%s/%s" % (root, section, prop), defaultValue)) + return S_OK(defaultValue) def isDownloadableGroup(groupName): @@ -667,24 +710,6 @@ def isDownloadableGroup(groupName): return True -def getUserDict(username): - """ Get full information from user section - - :param str username: DIRAC user name - - :return: S_OK()/S_ERROR() - """ - resDict = {} - relPath = '%s/Users/%s/' % (gBaseRegistrySection, username) - result = gConfig.getConfigurationTree(relPath) - if not result['OK']: - return result - for key, value in result['Value'].items(): - if value: - resDict[key.replace(relPath, '')] = value - return S_OK(resDict) - - def getEmailsForGroup(groupName): """ Get email list of users in group @@ -697,3 +722,110 @@ def getEmailsForGroup(groupName): email = getUserOption(username, 'Email', []) emails.append(email) return emails + + +def getIDsForUsername(username): + """ Return IDs for DIRAC user + + :param str username: DIRAC user + + :return: list -- contain IDs + """ + return gConfig.getValue("%s/Users/%s/ID" % (gBaseRegistrySection, username), []) + + +def getVOsWithVOMS(voList=None): + """ Get all the configured VOMS VOs + + :param list voList: VOs where to look + + :return: S_OK(list)/S_ERROR() + """ + vos = [] + if not voList: + result = getVOs() + if result['OK']: + voList = result['Value'] + for vo in voList or []: + if getVOOption(vo, 'VOMSName'): + vos.append(vo) + return S_OK(vos) + + +def getDNsForUsername(username): + """ Find all DNs for DIRAC user + + :param str username: DIRAC user + + :return: S_OK(list)/S_ERROR() -- contain DNs + """ + userDNs = getDNsForUsernameFromCS(username) + for uid in getIDsForUsername(username): + result = gAuthManagerData.getDNsForID(uid) + if result['OK']: + userDNs += result['Value'] + return S_OK(list(set(userDNs))) + + +def getDNForUsernameInGroup(username, group, checkStatus=False): + """ Get user DN for user in group + + :param str username: user name + :param str group: group name + :param bool checkStatus: don't add suspended DNs + + :return: S_OK(str)/S_ERROR() + """ + result = getDNsForUsernameInGroup(username, group, checkStatus) + return S_OK(result['Value'][0]) if result['OK'] else result + + +def getDNsForUsernameInGroup(username, group, checkStatus=False): + """ Get user DN for user in group + + :param str username: user name + :param str group: group name + :param bool checkStatus: don't add suspended DNs + + :return: S_OK(str)/S_ERROR() + """ + if username not in getGroupOption(group, 'Users', []): + return S_ERROR('%s group not have %s user.' % (group, username)) + DNs = [] + result = getDNsForUsername(username) + if not result['OK']: + return result + userDNs = result['Value'] + print('== getDNsForUsernameInGroup ==') + pprint(userDNs) + + vo = getGroupOption(group, 'VO') + if checkStatus and vo in getUserOption(username, 'Suspended', []): + return S_ERROR('%s marked as suspended for %s VO.' % (username, vo)) + result = getVOsWithVOMS([vo]) + if not result['OK']: + return result + if result['Value']: + result = getVOMSInfo(vo=vo) + if not result['OK']: + return result + vomsData = result['Value'] + if vomsData.get(vo) and vomsData[vo]['OK']: + voData = vomsData[vo]['Value'] + role = getGroupOption(group, 'VOMSRole') + for dn in userDNs: + if dn in voData and (not checkStatus or not voData[dn]['Suspended']): + if not role or role in voData[dn]['ActiveRoles' if checkStatus else 'VOMSRoles']: + DNs.append(dn) + else: + DNs += userDNs + else: + DNs += userDNs + print('-------------------------') + pprint(DNs) + dns = list(set([e for e in DNs if e])) + print('-------------------------') + pprint(dns) + if dns: + return S_OK(list(set(dns))) + return S_ERROR('For %s@%s not found DN%s.' % (username, group, ' or it suspended' if checkStatus else '')) diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py index 84455a045f4..256a80f7cf9 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py @@ -428,45 +428,68 @@ def getFilterConfig(filterID): return gConfig.getOptionsDict('Resources/LogFilters/%s' % filterID) -def getInfoAboutProviders(of=None, providerName=None, option='', section=''): - """ Get the information about providers - - :param str of: provider of what(Id, Proxy or etc.) need to look, - None, "all" to get list of instance of what this providers - :param str providerName: provider name, - None, "all" to get list of providers names - :param str option: option name that need to get, - None, "all" to get all options in a section - :param str section: section path in root section of provider, - "all" to get options in all sections - - :return: S_OK()/S_ERROR() +def getProvidersForInstance(instance, providerType=None): + """ Get providers for instance + + :param str instance: instance of what this providers + :param str providerType: provider type + + :return: S_OK(list)/S_ERROR() """ - if not of or of == "all": + result = gConfig.getSections('%s/%sProviders' % (gBaseResourcesSection, instance)) + if not result['OK'] or not result['Value'] or (result['OK'] and not providerType): + return result + data = [] + for prov in result['Value']: + if providerType == gConfig.getValue('%s/%sProviders/%s/ProviderType' % + (gBaseResourcesSection, instance, prov)): + data.append(prov) + return S_OK(data) + + +def getProviderByAlias(alias, instance=None): + """ Find provider name by alias + + :param str alias: other registered provider name + :param str instance: provider of what + + :return: S_OK(str)/S_ERROR() + """ + instances = [instance] or [] + if not instances: result = gConfig.getSections(gBaseResourcesSection) if not result['OK']: return result - return S_OK([i.replace('Providers', '') for i in result['Value']]) - if not providerName or providerName == "all": - return gConfig.getSections('%s/%sProviders' % (gBaseResourcesSection, of)) - if not option or option == 'all': - if not section: - return gConfig.getOptionsDict( - "%s/%sProviders/%s" % (gBaseResourcesSection, of, providerName)) - elif section == "all": - resDict = {} - relPath = "%s/%sProviders/%s/" % (gBaseResourcesSection, of, providerName) - result = gConfig.getConfigurationTree(relPath) + for section in result['Value']: + if section.endswith('Providers'): + instances.append(section.rsplit('Providers', 1)[0]) + for instance in instances: + result = getProvidersForInstance(instance) + if not result['OK']: + return result + for provider in result['Value']: + if alias in gConfig.getValue("%s/%sProviders/%s/Aliases" % (gBaseResourcesSection, + instance, provider), []): + return S_OK(provider) + return S_ERROR('Did not find any provider for %s' % alias) + + +def getProviderInfo(provider): + """ Get provider info + + :param str provider: provider + + :return: S_OK(dict)/S_ERROR() + """ + result = gConfig.getSections(gBaseResourcesSection) + if not result['OK']: + return result + for section in result['Value']: + if section.endswith('Providers'): + result = getProvidersForInstance(section[:-9]) if not result['OK']: return result - for key, value in result['Value'].items(): # can be an iterator - if value: - resDict[key.replace(relPath, '')] = value - return S_OK(resDict) - else: - return gConfig.getSections( - '%s/%sProviders/%s/%s/' % (gBaseResourcesSection, of, providerName, section)) - else: - return S_OK(gConfig.getValue( - '%s/%sProviders/%s/%s/%s' % (gBaseResourcesSection, of, providerName, - section, option))) + if provider in result['Value']: + return gConfig.getOptionsDictRecursively("%s/%s/%s/" % (gBaseResourcesSection, + section, provider)) + return S_ERROR('%s provider not found.' % provider) diff --git a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py index 4a748aad28e..d065dd4b6c0 100755 --- a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py +++ b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py @@ -66,6 +66,8 @@ def getComponentSection(componentName, componentTuple=False, setup=False, compon systemSection = getSystemSection(componentName, componentTuple, setup=setup) return "%s/%s/%s" % (systemSection, componentCategory, componentTuple[1]) +def getAPISection(APIName, APITuple=False, setup=False): + return getComponentSection(APIName, APITuple, setup, "APIs") def getServiceSection(serviceName, serviceTuple=False, setup=False): return getComponentSection(serviceName, serviceTuple, setup, "Services") diff --git a/src/DIRAC/ConfigurationSystem/Client/Utilities.py b/src/DIRAC/ConfigurationSystem/Client/Utilities.py index d4b9cef6bab..6d522726b2b 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Utilities.py +++ b/src/DIRAC/ConfigurationSystem/Client/Utilities.py @@ -536,14 +536,14 @@ def getElasticDBParameters(fullname): return S_OK(parameters) -def getOAuthAPI(instance='Production'): - """ Get OAuth API url +def getAuthAPI(instance='Production'): + """ Get Auth REST API url :param str instance: instance :return: str """ - return gConfig.getValue("/Systems/Framework/%s/URLs/OAuthAPI" % instance) + return gConfig.getValue("/Systems/Framework/%s/URLs/AuthAPI" % instance) def getDIRACGOCDictionary(): diff --git a/src/DIRAC/ConfigurationSystem/test/Test_agentOptions.py b/src/DIRAC/ConfigurationSystem/test/Test_agentOptions.py index 20c551e7725..c7a5ce34598 100644 --- a/src/DIRAC/ConfigurationSystem/test/Test_agentOptions.py +++ b/src/DIRAC/ConfigurationSystem/test/Test_agentOptions.py @@ -68,7 +68,7 @@ 'Enable']}), ('DIRAC.WorkloadManagementSystem.Agent.StatesAccountingAgent', {}), ('DIRAC.WorkloadManagementSystem.Agent.SiteDirector', - {'SpecialMocks': {'findGenericPilotCredentials': S_OK(('a', 'b'))}}), + {'SpecialMocks': {'findGenericPilotCredentials': S_OK(('a', 'b', 'c'))}}), ] diff --git a/src/DIRAC/Core/Base/API.py b/src/DIRAC/Core/Base/API.py index c01f56fd82a..1dea9fa893d 100644 --- a/src/DIRAC/Core/Base/API.py +++ b/src/DIRAC/Core/Base/API.py @@ -9,6 +9,7 @@ from DIRAC import gLogger, gConfig, S_OK, S_ERROR from DIRAC.Core.Security.ProxyInfo import getProxyInfo, formatProxyInfoAsString +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNsForUsername from DIRAC.Core.Utilities.Version import getCurrentVersion from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites @@ -159,9 +160,11 @@ def _getCurrentUser(self): gLogger.debug(formatProxyInfoAsString(proxyInfo)) if 'group' not in proxyInfo: return self._errorReport('Proxy information does not contain the group', res['Message']) - res = getDNForUsername(proxyInfo['username']) - if not res['OK']: - return self._errorReport('Failed to get proxies for user', res['Message']) + result = getDNsForUsername(proxyInfo['username']) + if not result['OK']: + return self._errorReport('Failed to get proxies for user', result['Message']) + if not result['Value']: + return self._errorReport('Failed to get proxies for user', "No DNs found for %s" % proxyInfo['username']) return S_OK(proxyInfo['username']) ############################################################################# diff --git a/src/DIRAC/Core/Base/DB.py b/src/DIRAC/Core/Base/DB.py index af4b9af4fc2..e182be07f90 100755 --- a/src/DIRAC/Core/Base/DB.py +++ b/src/DIRAC/Core/Base/DB.py @@ -5,7 +5,7 @@ from __future__ import division from __future__ import print_function -from DIRAC.Core.Base.DIRACDB import DIRACDB +from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities.MySQL import MySQL from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters @@ -17,8 +17,15 @@ class DB(DIRACDB, MySQL): """ def __init__(self, dbname, fullname, debug=False): + """ C'or + :param str dbname: database name + :param str fullname: full name + :param bool debug: debug mode + """ + self.versionDB = 0 self.fullname = fullname + self.versionTable = '%s_Version' % dbname result = getDBParameters(fullname) if not result['OK']: @@ -41,10 +48,43 @@ def __init__(self, dbname, fullname, debug=False): if not self._connected: raise RuntimeError("Can not connect to DB '%s', exiting..." % self.dbName) + # Initialize version + result = self._query("show tables") + if result['OK']: + if self.versionTable not in [t[0] for t in result['Value']]: + result = self._createTables({self.versionTable: {'Fields': {'Version': 'INTEGER NOT NULL'}, + 'PrimaryKey': 'Version'}}) + if not result['OK']: + raise RuntimeError("Can not initialize %s version: %s" % (self.dbName, result['Message'])) + result = self._query("SELECT Version FROM `%s`" % self.versionTable) + if result['OK']: + if len(result['Value']) > 0: + self.versionDB = result['Value'][0][0] + else: + result = self._update("INSERT INTO `%s` (Version) VALUES (%s)" % (self.versionTable, self.versionDB)) + if not result['OK']: + raise RuntimeError("Can not initialize %s version: %s" % (self.dbName, result['Message'])) + self.log.info("===================== MySQL ======================") self.log.info("User: " + self.dbUser) self.log.info("Host: " + self.dbHost) self.log.info("Port: " + str(self.dbPort)) - # self.log.info("Password: "+ self.dbPass) + #self.log.info("Password: "+self.dbPass) self.log.info("DBName: " + self.dbName) self.log.info("==================================================") + + + def updateDBVersion(self, version): + """ Update DB version + + :param int version: version number + + :return: S_OK()/S_ERROR() + """ + result = self._query('DELETE FROM `%s_Version`' % self.dbName) + if result['OK']: + result = self._update("INSERT INTO `%s_Version` (Version) VALUES (%s)" % (self.dbName, version)) + if not result['OK']: + return S_ERROR("Can not initialize %s version: %s" % (self.dbName, result['Message'])) + self.versionDB = version + return S_OK() diff --git a/src/DIRAC/Core/DISET/AuthManager.py b/src/DIRAC/Core/DISET/AuthManager.py index 847043cc987..8304d37bb41 100755 --- a/src/DIRAC/Core/DISET/AuthManager.py +++ b/src/DIRAC/Core/DISET/AuthManager.py @@ -5,180 +5,249 @@ from __future__ import print_function import six + +from DIRAC.Core.Utilities import List +from DIRAC.Core.Security import Properties +from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.Helpers import Registry -from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Security import Properties -from DIRAC.Core.Utilities import List __RCSID__ = "$Id$" +KW_ID = 'ID' +KW_DN = 'DN' +KW_GROUP = 'group' +KW_USERNAME = 'username' +KW_HOSTS_GROUP = 'hosts' +KW_PROPERTIES = 'properties' +KW_EXTRA_CREDENTIALS = 'extraCredentials' + + +def forwardingCredentials(credDict, logObj=gLogger): + """ Check whether the credentials are being forwarded by a valid source and extract it + + :param dict credDict: Credentials to ckeck + :param object logObj: logger + + :return: bool + """ + if isinstance(credDict.get(KW_EXTRA_CREDENTIALS), (tuple, list)): + retVal = Registry.getHostnameForDN(credDict.get(KW_DN)) + if not retVal['OK']: + logObj.debug("The credentials forwarded not by a host:", credDict.get(KW_DN)) + return False + hostname = retVal['Value'] + if Properties.TRUSTED_HOST not in Registry.getPropertiesForHost(hostname, []): + logObj.debug("The credentials forwarded by a %s host, but it is not a trusted one" % hostname) + return False + if credDict[KW_EXTRA_CREDENTIALS][0][0] == '/': + credDict[KW_DN] = credDict[KW_EXTRA_CREDENTIALS][0] + credDict[KW_ID] = None + else: + credDict[KW_ID] = credDict[KW_EXTRA_CREDENTIALS][0] + credDict[KW_DN] = None + credDict[KW_GROUP] = credDict[KW_EXTRA_CREDENTIALS][1] + del credDict[KW_EXTRA_CREDENTIALS] + return True + return False + + +def authorizeBySession(credDict, logObj=gLogger): + """ Discover the username associated to the authentication session. It will check if the selected group is valid. + The username will be included in the credentials dictionary. And will discover DN for group if last not set. + + :param dict credDict: Credentials to check + :param object logObj: logger + + :return: bool -- specifying whether the username was found + """ + # Find user + result = Registry.getUsernameForID(credDict[KW_ID]) + if not result['OK']: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + credDict[KW_USERNAME] = result['Value'] + return __checkGroup(credDict, logObj=gLogger) + + +def authorizeByCertificate(credDict, logObj=gLogger): + """ Discover the username associated to the certificate DN. It will check if the selected group is valid. + The username will be included in the credentials dictionary. + + :param dict credDict: Credentials to check + :param object logObj: logger + + :return: bool -- specifying whether the username was found + """ + # Search host + result = Registry.getHostnameForDN(credDict[KW_DN]) + if result['OK'] and result['Value']: + credDict[KW_USERNAME] = result['Value'] + credDict[KW_GROUP] = KW_HOSTS_GROUP + return __checkGroup(credDict, logObj=gLogger) + elif credDict.get(KW_GROUP) == KW_HOSTS_GROUP: + logObj.warn("Cannot find hostname for DN %s: %s" % (credDict[KW_DN], result['Message'])) + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + + # Search user + result = Registry.getUsernameForDN(credDict[KW_DN]) + if not result['OK']: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + credDict[KW_USERNAME] = result['Value'] + return __checkGroup(credDict, logObj=gLogger) + + +def __checkGroup(credDict, logObj=gLogger): + """ Check/get default group + + :param dict credDict: Credentials to check + :param object logObj: logger + + :return: bool -- specifying whether the username was found + """ + # Find/check group + credDict[KW_PROPERTIES] = [] + if not credDict.get(KW_GROUP) or credDict[KW_GROUP] == 'visitor': + result = Registry.findDefaultGroupForUser(credDict[KW_USERNAME]) + if not result['OK']: + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + credDict[KW_GROUP] = result['Value'] + if credDict[KW_GROUP] == KW_HOSTS_GROUP: + credDict[KW_PROPERTIES] = Registry.getPropertiesForHost(credDict[KW_USERNAME], []) + return True + if not Registry.getGroupsForUser(credDict[KW_USERNAME], groupsList=[credDict[KW_GROUP]]).get('Value'): + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + return False + # Get DN for user/group + result = Registry.getDNsForUsernameInGroup(credDict[KW_USERNAME], credDict[KW_GROUP], checkStatus=True) + if not result['OK']: + logObj.error(result['Message']) + credDict[KW_GROUP] = "visitor" + return False + # Set DN if authorization not througth certificate + if not credDict.get(KW_DN): + credDict[KW_DN] = result['Value'][0] + # Check if DN match for group + if credDict[KW_DN] not in result["Value"]: + logObj.error('%s DN is not match for %s group.' % (credDict[KW_DN], credDict[KW_GROUP])) + credDict[KW_GROUP] = "visitor" + return False + + # Fill group properties + credDict[KW_PROPERTIES] = Registry.getPropertiesForGroup(credDict[KW_GROUP], []) + return True + class AuthManager(object): """ Handle Service Authorization """ - __authLogger = gLogger.getSubLogger("Authorization") - KW_HOSTS_GROUP = 'hosts' - KW_DN = 'DN' - KW_GROUP = 'group' - KW_EXTRA_CREDENTIALS = 'extraCredentials' - KW_PROPERTIES = 'properties' - KW_USERNAME = 'username' def __init__(self, authSection): - """ - Constructor + """ Constructor - :type authSection: string - :param authSection: Section containing the authorization rules + :param str authSection: Section containing the authorization rules """ self.authSection = authSection def authQuery(self, methodQuery, credDict, defaultProperties=False): - """ - Check if the query is authorized for a credentials dictionary - - :type methodQuery: string - :param methodQuery: Method to test - :type credDict: dictionary - :param credDict: dictionary containing credentials for test. The dictionary can contain the DN - and selected group. - :return: Boolean result of test + """ Check if the query is authorized for a credentials dictionary + + :param str methodQuery: Method to test + :param dict credDict: dictionary containing credentials for test. The dictionary can contain the DN + and selected group. + :param defaultProperties: default properties + :type defaultProperties: list or tuple + + :return: bool -- result of test """ userString = "" - if self.KW_DN in credDict: - userString += "DN=%s" % credDict[self.KW_DN] - if self.KW_GROUP in credDict: - userString += " group=%s" % credDict[self.KW_GROUP] - if self.KW_EXTRA_CREDENTIALS in credDict: - userString += " extraCredentials=%s" % str(credDict[self.KW_EXTRA_CREDENTIALS]) + if KW_ID in credDict: + userString += " ID=%s" % credDict[KW_ID] + if KW_DN in credDict: + userString += " DN=%s" % credDict[KW_DN] + if credDict.get(KW_GROUP): + userString += " group=%s" % credDict[KW_GROUP] + if KW_EXTRA_CREDENTIALS in credDict: + userString += " extraCredentials=%s" % str(credDict[KW_EXTRA_CREDENTIALS]) self.__authLogger.debug("Trying to authenticate %s" % userString) + # Get properties requiredProperties = self.getValidPropertiesForMethod(methodQuery, defaultProperties) + lowerCaseProperties = [prop.lower() for prop in requiredProperties] or ['any'] + allowAll = "any" in lowerCaseProperties or "all" in lowerCaseProperties + # Extract valid groups validGroups = self.getValidGroups(requiredProperties) - lowerCaseProperties = [prop.lower() for prop in requiredProperties] - if not lowerCaseProperties: - lowerCaseProperties = ['any'] + self.__authLogger.info("validGroups: ", validGroups) - allowAll = "any" in lowerCaseProperties or "all" in lowerCaseProperties - # Set no properties by default - credDict[self.KW_PROPERTIES] = [] - # Check non secure backends - if self.KW_DN not in credDict or not credDict[self.KW_DN]: - if allowAll and not validGroups: - self.__authLogger.debug("Accepted request from unsecure transport") - return True + # Read extra credentials + if KW_EXTRA_CREDENTIALS in credDict: + # Is it a host? and HACK TO MAINTAIN COMPATIBILITY + if credDict.get(KW_EXTRA_CREDENTIALS) == KW_HOSTS_GROUP: + credDict[KW_GROUP] = credDict[KW_EXTRA_CREDENTIALS] + del credDict[KW_EXTRA_CREDENTIALS] + # Check if query comes though a gateway/web server + elif forwardingCredentials(credDict, logObj=self.__authLogger): + self.__authLogger.debug("Query comes from a gateway") + return self.authQuery(methodQuery, credDict, requiredProperties) else: - self.__authLogger.debug( - "Explicit property required and query seems to be coming through an unsecure transport") return False - # Check if query comes though a gateway/web server - if self.forwardedCredentials(credDict): - self.__authLogger.debug("Query comes from a gateway") - self.unpackForwardedCredentials(credDict) - return self.authQuery(methodQuery, credDict, requiredProperties) - # Get the properties - # Check for invalid forwarding - if self.KW_EXTRA_CREDENTIALS in credDict: - # Invalid forwarding? - if not isinstance(credDict[self.KW_EXTRA_CREDENTIALS], six.string_types): - self.__authLogger.debug("The credentials seem to be forwarded by a host, but it is not a trusted one") - return False - # Is it a host? - if self.KW_EXTRA_CREDENTIALS in credDict and credDict[self.KW_EXTRA_CREDENTIALS] == self.KW_HOSTS_GROUP: - # Get the nickname of the host - credDict[self.KW_GROUP] = credDict[self.KW_EXTRA_CREDENTIALS] - # HACK TO MAINTAIN COMPATIBILITY - else: - if self.KW_EXTRA_CREDENTIALS in credDict and self.KW_GROUP not in credDict: - credDict[self.KW_GROUP] = credDict[self.KW_EXTRA_CREDENTIALS] - # END OF HACK - # Get the username - if self.KW_DN in credDict and credDict[self.KW_DN]: - if self.KW_GROUP not in credDict: - result = Registry.findDefaultGroupForDN(credDict[self.KW_DN]) - if not result['OK']: - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - else: - credDict[self.KW_GROUP] = result['Value'] - if credDict[self.KW_GROUP] == self.KW_HOSTS_GROUP: - # For host - if not self.getHostNickName(credDict): - self.__authLogger.warn("Host is invalid") - if not allowAll: - return False - # If all, then set anon credentials - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" + + authorized = True + # User/group authorization + if not credDict.get(KW_USERNAME): + if credDict.get(KW_DN): + # With certificate + authorized = authorizeByCertificate(credDict, logObj=self.__authLogger) + elif credDict.get(KW_ID): + # With IdP session + authorized = authorizeBySession(credDict, logObj=self.__authLogger) else: - # For users - username = self.getUsername(credDict) - suspended = self.isUserSuspended(credDict) - if not username: - self.__authLogger.warn("User is invalid or does not belong to the group it's saying") - if suspended: - self.__authLogger.warn("User is Suspended") - - if not username or suspended: - if not allowAll: - return False - # If all, then set anon credentials - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" - else: - if not allowAll: - return False - credDict[self.KW_USERNAME] = "anonymous" - credDict[self.KW_GROUP] = "visitor" + credDict[KW_USERNAME] = "anonymous" + credDict[KW_GROUP] = "visitor" + authorized = False - # If any or all in the props, allow - allowGroup = not validGroups or credDict[self.KW_GROUP] in validGroups - if allowAll and allowGroup: - return True - # Check authorized groups - if "authenticated" in lowerCaseProperties and allowGroup: - return True - if not self.matchProperties(credDict, requiredProperties): + # Access to the service + + # Check free access + if not allowAll and not authorized: + self.__authLogger.debug("User is invalid or does not belong to the group it's saying") + return False + # Match properties + if not self.matchProperties(credDict, list(set(requiredProperties) - set(['Any', 'any', + 'All', 'all', + 'authenticated', + 'Authenticated']))): self.__authLogger.warn("Client is not authorized\nValid properties: %s\nClient: %s" % - (requiredProperties, credDict)) + (requiredProperties, credDict)) return False - elif not allowGroup: + # Match allowed groups + if validGroups and credDict[KW_GROUP] not in validGroups: self.__authLogger.warn("Client is not authorized\nValid groups: %s\nClient: %s" % - (validGroups, credDict)) + (validGroups, credDict)) return False - return True - - def getHostNickName(self, credDict): - """ - Discover the host nickname associated to the DN. - The nickname will be included in the credentials dictionary. - - :type credDict: dictionary - :param credDict: Credentials to ckeck - :return: Boolean specifying whether the nickname was found - """ - if self.KW_DN not in credDict: - return True - if self.KW_GROUP not in credDict: - return False - retVal = Registry.getHostnameForDN(credDict[self.KW_DN]) - if not retVal['OK']: - gLogger.warn("Cannot find hostname for DN %s: %s" % (credDict[self.KW_DN], retVal['Message'])) - return False - credDict[self.KW_USERNAME] = retVal['Value'] - credDict[self.KW_PROPERTIES] = Registry.getPropertiesForHost(credDict[self.KW_USERNAME], []) + # Access allowed + if not authorized: + self.__authLogger.debug("Accepted request from unsecure transport") return True def getValidPropertiesForMethod(self, method, defaultProperties=False): - """ - Get all authorized groups for calling a method + """ Get all authorized groups for calling a method + + :param str method: Method to test + :param defaultProperties: default properties + :type defaultProperties: list or tuple - :type method: string - :param method: Method to test - :return: List containing the allowed groups + :return: list -- List containing the allowed groups """ authProps = gConfig.getValue("%s/%s" % (self.authSection, method), []) if authProps: @@ -197,11 +266,11 @@ def getValidPropertiesForMethod(self, method, defaultProperties=False): return [] def getValidGroups(self, rawProperties): - """ Get valid groups as specified in the method authorization rules + """ Get valid groups as specified in the method authorization rules - :param rawProperties: all method properties - :type rawProperties: python:list - :return: list of allowed groups or [] + :param list rawProperties: all method properties + + :return: list -- list of allowed groups """ validGroups = [] for prop in list(rawProperties): @@ -219,103 +288,26 @@ def getValidGroups(self, rawProperties): validGroups = list(set(validGroups)) return validGroups - def forwardedCredentials(self, credDict): - """ - Check whether the credentials are being forwarded by a valid source - - :type credDict: dictionary - :param credDict: Credentials to ckeck - :return: Boolean with the result - """ - if self.KW_EXTRA_CREDENTIALS in credDict and isinstance(credDict[self.KW_EXTRA_CREDENTIALS], (tuple, list)): - if self.KW_DN in credDict: - retVal = Registry.getHostnameForDN(credDict[self.KW_DN]) - if retVal['OK']: - hostname = retVal['Value'] - if Properties.TRUSTED_HOST in Registry.getPropertiesForHost(hostname, []): - return True - return False - - def unpackForwardedCredentials(self, credDict): - """ - Extract the forwarded credentials - - :type credDict: dictionary - :param credDict: Credentials to unpack - """ - credDict[self.KW_DN] = credDict[self.KW_EXTRA_CREDENTIALS][0] - credDict[self.KW_GROUP] = credDict[self.KW_EXTRA_CREDENTIALS][1] - del(credDict[self.KW_EXTRA_CREDENTIALS]) - - def getUsername(self, credDict): - """ - Discover the username associated to the DN. It will check if the selected group is valid. - The username will be included in the credentials dictionary. - - :type credDict: dictionary - :param credDict: Credentials to check - :return: Boolean specifying whether the username was found - """ - if self.KW_DN not in credDict: - return True - if self.KW_GROUP not in credDict: - result = Registry.findDefaultGroupForDN(credDict[self.KW_DN]) - if not result['OK']: - return False - credDict[self.KW_GROUP] = result['Value'] - credDict[self.KW_PROPERTIES] = Registry.getPropertiesForGroup(credDict[self.KW_GROUP], []) - usersInGroup = Registry.getUsersInGroup(credDict[self.KW_GROUP], []) - if not usersInGroup: - return False - retVal = Registry.getUsernameForDN(credDict[self.KW_DN], usersInGroup) - if retVal['OK']: - credDict[self.KW_USERNAME] = retVal['Value'] - return True - return False + def matchProperties(self, credDict, validProps, caseSensitive=False): + """ Return True if one or more properties are in the valid list of properties - def isUserSuspended(self, credDict): - """ Discover if the user is in Suspended status + :param dict credDict: credentials to match + :param list validProps: List of valid properties + :param bool caseSensitive: Map lower case properties to properties to make the check in + lowercase but return the proper case - :param dict credDict: Credentials to check - :return: Boolean True if user is Suspended + :return: bool -- specifying whether any property has matched the valid ones """ - # Update credDict if the username is not there - if self.KW_USERNAME not in credDict: - self.getUsername(credDict) - # If username or group is not known we can not judge if the user is suspended - # These cases are treated elsewhere anyway - if self.KW_USERNAME not in credDict or self.KW_GROUP not in credDict: - return False - suspendedVOList = Registry.getUserOption(credDict[self.KW_USERNAME], 'Suspended', []) - if not suspendedVOList: - return False - vo = Registry.getVOForGroup(credDict[self.KW_GROUP]) - if vo in suspendedVOList: + if not validProps: return True - return False - - def matchProperties(self, credDict, validProps, caseSensitive=False): - """ - Return True if one or more properties are in the valid list of properties - - :type props: list - :param props: List of properties to match - :type validProps: list - :param validProps: List of valid properties - :return: Boolean specifying whether any property has matched the valid ones - """ - - # HACK: Map lower case properties to properties to make the check in lowercase but return the proper case if not caseSensitive: validProps = dict((prop.lower(), prop) for prop in validProps) else: validProps = dict((prop, prop) for prop in validProps) - groupProperties = credDict[self.KW_PROPERTIES] foundProps = [] - for prop in groupProperties: + for prop in credDict[KW_PROPERTIES]: if not caseSensitive: prop = prop.lower() if prop in validProps: foundProps.append(validProps[prop]) - credDict[self.KW_PROPERTIES] = foundProps - return foundProps + return bool(foundProps) diff --git a/src/DIRAC/Core/DISET/RequestHandler.py b/src/DIRAC/Core/DISET/RequestHandler.py index 8715f323162..7c47c867105 100755 --- a/src/DIRAC/Core/DISET/RequestHandler.py +++ b/src/DIRAC/Core/DISET/RequestHandler.py @@ -10,7 +10,6 @@ import six import time import psutil -import six import DIRAC @@ -20,6 +19,8 @@ from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.Core.Security.Properties import CS_ADMINISTRATOR +from DIRAC.Core.DISET.AuthManager import authorizeByCertificate,\ + forwardingCredentials, authorizeBySession def getServiceOption(serviceInfo, optionName, defaultValue): @@ -100,7 +101,13 @@ def getRemoteCredentials(self): :return: Credentials dictionary of remote peer. """ - return self.__trPool.get(self.__trid).getConnectingCredentials() + credDict = self.__trPool.get(self.__trid).getConnectingCredentials() + forwardingCredentials(credDict, logObj=self.log) + if credDict.get('ID'): + authorizeBySession(credDict, logObj=self.log) + else: + authorizeByCertificate(credDict, logObj=self.log) + return credDict @classmethod def getCSOption(cls, optionName, defaultValue=False): diff --git a/src/DIRAC/Core/DISET/ThreadConfig.py b/src/DIRAC/Core/DISET/ThreadConfig.py index 77d3d19e89d..dfff5e4ccc0 100644 --- a/src/DIRAC/Core/DISET/ThreadConfig.py +++ b/src/DIRAC/Core/DISET/ThreadConfig.py @@ -27,6 +27,7 @@ def reset(self): """ Reset extra information """ self.__DN = False + self.__ID = False self.__group = False self.__deco = False self.__setup = False @@ -73,21 +74,19 @@ def getGroup(self): """ return self.__group - def setID(self, DN, group): + def setID(self, ID): """ Set user ID - :param str DN: user DN - :param str group: user group + :param str ID: user ID """ - self.__DN = DN - self.__group = group + self.__ID = ID def getID(self): """ Return user ID - :return: tuple + :return: str """ - return (self.__DN, self.__group) + return self.__ID def setSetup(self, setup): """ Set setup name @@ -108,19 +107,17 @@ def dump(self): :return: tuple """ - return (self.__DN, self.__group, self.__setup) + return (self.__DN, self.__group, self.__setup, self.__ID) def load(self, tp): """ Save extra information - :param tuple tp: contain DN, group name, setup name + :param tuple tp: contains DN, group name, setup name, userID """ - if tp[0]: - self.__DN = tp[0] - if tp[1]: - self.__group = tp[1] - if tp[2]: - self.__setup = tp[2] + self.__ID = tp[3] or self.__ID + self.__DN = tp[0] or self.__DN + self.__group = tp[1] or self.__group + self.__setup = tp[2] or self.__setup def threadDeco(method): diff --git a/src/DIRAC/Core/DISET/private/BaseClient.py b/src/DIRAC/Core/DISET/private/BaseClient.py index 7df81ad7bba..f49b3b46769 100755 --- a/src/DIRAC/Core/DISET/private/BaseClient.py +++ b/src/DIRAC/Core/DISET/private/BaseClient.py @@ -15,11 +15,10 @@ import DIRAC from DIRAC.Core.DISET.private.Protocols import gProtocolDict from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Utilities import List, Network +from DIRAC.Core.Utilities import List, Network, DErrno from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL, getServiceFailoverURL -from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck from DIRAC.Core.DISET.private.TransportPool import getGlobalTransportPool from DIRAC.Core.DISET.ThreadConfig import ThreadConfig @@ -37,6 +36,7 @@ class BaseClient(object): KW_TIMEOUT = "timeout" KW_SETUP = "setup" KW_VO = "VO" + KW_DELEGATED_ID = "delegatedID" KW_DELEGATED_DN = "delegatedDN" KW_DELEGATED_GROUP = "delegatedGroup" KW_IGNORE_GATEWAYS = "ignoreGateways" @@ -259,17 +259,19 @@ def __discoverExtraCredentials(self): self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS] # Are we delegating something? + delegatedID = self.kwargs.get(self.KW_DELEGATED_ID) or self.__threadConfig.getID() delegatedDN = self.kwargs.get(self.KW_DELEGATED_DN) or self.__threadConfig.getDN() delegatedGroup = self.kwargs.get(self.KW_DELEGATED_GROUP) or self.__threadConfig.getGroup() + + if delegatedID: + self.kwargs[self.KW_DELEGATED_ID] = delegatedID if delegatedDN: self.kwargs[self.KW_DELEGATED_DN] = delegatedDN - if not delegatedGroup: - result = Registry.findDefaultGroupForDN(delegatedDN) - if not result['OK']: - return result - delegatedGroup = result['Value'] + if delegatedGroup: self.kwargs[self.KW_DELEGATED_GROUP] = delegatedGroup - self.__extraCredentials = (delegatedDN, delegatedGroup) + + if delegatedID or delegatedDN: + self.__extraCredentials = (delegatedID or delegatedDN, delegatedGroup) return S_OK() def __findServiceURL(self): @@ -506,10 +508,10 @@ def _connect(self): # try to reconnect return self._connect() else: - return retVal + return S_ERROR(DErrno.ECONNECT, retVal['Message']) except Exception as e: gLogger.exception(lException=True, lExcInfo=True) - return S_ERROR("Can't connect to %s: %s" % (self.serviceURL, repr(e))) + return S_ERROR(DErrno.ECONNECT, "Can't connect to %s: %s" % (self.serviceURL, repr(e))) # We add the connection to the transport pool gLogger.debug("Connected to: %s" % self.serviceURL) trid = getGlobalTransportPool().add(transport) diff --git a/src/DIRAC/Core/DISET/test/Test_AuthManager.py b/src/DIRAC/Core/DISET/test/Test_AuthManager.py index 56645994f00..814f6998b96 100644 --- a/src/DIRAC/Core/DISET/test/Test_AuthManager.py +++ b/src/DIRAC/Core/DISET/test/Test_AuthManager.py @@ -4,14 +4,44 @@ from __future__ import division from __future__ import print_function +import os +import mock +import pickle import unittest from diraccfg import CFG -from DIRAC import gConfig +from DIRAC import gConfig, rootPath, S_OK, S_ERROR from DIRAC.Core.DISET.AuthManager import AuthManager __RCSID__ = "$Id$" +workDir = os.path.join(gConfig.getValue('/LocalSite/InstancePath', rootPath), 'work/ProxyManager') + +voDict = { + 'testVO': S_OK({ + '/User/test/DN/CN=userS': { + 'Suspended': True, + 'VOMSRoles': [u'/testVO'], + 'ActiveRoles': [], + 'SuspendedRoles': [u'/testVO'] + }, + '/User/test/DN/CN=userA': { + 'Suspended': False, + 'VOMSRoles': [u'/testVO'], + 'ActiveRoles': [u'/testVO'], + 'SuspendedRoles': [] + } + }), + 'testVOOther': S_OK({ + '/User/test/DN/CN=userS': { + 'Suspended': False, + 'VOMSRoles': [u'/testVOOther'], + 'ActiveRoles': [u'/testVOOther'], + 'SuspendedRoles': [] + } + }) +} + testSystemsCFG = """ Systems { @@ -46,6 +76,7 @@ testVO { VOAdmin = userA + VOMSName = testVO } testVOBad { @@ -54,6 +85,7 @@ testVOOther { VOAdmin = userA + VOMSName = testVOOther } } Users @@ -91,6 +123,7 @@ { Users = userA, userS VO = testVO + VOMSRole = /testVO Properties = NormalUser } group_test_other @@ -110,9 +143,28 @@ """ +def sf_getVOMSInfo(vo=None, dn=None): + return S_OK(voDict) + + +@mock.patch('DIRAC.ConfigurationSystem.Client.Helpers.Registry.getVOMSInfo', + new=sf_getVOMSInfo) class AuthManagerTest(unittest.TestCase): """ Base class for the Modules test cases """ + @classmethod + def setUpClass(cls): + if not os.path.exists(workDir): + os.makedirs(workDir) + for vo, infoDict in voDict.items(): + with open(os.path.join(workDir, vo + '.pkl'), 'wb+') as f: + pickle.dump(infoDict, f, pickle.HIGHEST_PROTOCOL) + + @classmethod + def tearDownClass(cls): + for vo in voDict.keys(): + if os.path.exists(os.path.join(workDir, vo + '.pkl')): + os.remove(os.path.join(workDir, vo + '.pkl')) def setUp(self): self.authMgr = AuthManager('/Systems/Service/Authorization') diff --git a/src/DIRAC/Core/Security/Locations.py b/src/DIRAC/Core/Security/Locations.py index fe9e575aca6..0cd8b7808c6 100644 --- a/src/DIRAC/Core/Security/Locations.py +++ b/src/DIRAC/Core/Security/Locations.py @@ -29,7 +29,19 @@ def getProxyLocation(): # No gridproxy found return False -# Retrieve CA's location + +def getPrivateKeyLocation(): + """ Get the path of the currently active private key(for auth) + """ + # Grid-Security + retVal = gConfig.getOption('%s/Grid-Security' % g_SecurityConfPath) + if retVal['OK']: + keyPath = "%s/private.pem" % retVal['Value'] + if os.path.isfile(keyPath): + return keyPath + + # No private key found + return False def getCAsLocation(): diff --git a/src/DIRAC/Core/Security/VOMSService.py b/src/DIRAC/Core/Security/VOMSService.py index f949b5853aa..667319c9b6c 100644 --- a/src/DIRAC/Core/Security/VOMSService.py +++ b/src/DIRAC/Core/Security/VOMSService.py @@ -49,7 +49,7 @@ def attGetUserNickname(self, dn, _ca=None): :param str dn: user DN :param str _ca: CA, kept for backward compatibility - :return: S_OK with Value: nickname + :return: S_OK with Value: nickname """ if self.userDict is None: @@ -65,16 +65,18 @@ def attGetUserNickname(self, dn, _ca=None): return S_ERROR(DErrno.EVOMS, "No nickname defined") return S_OK(nickname) - def getUsers(self): + def getUsers(self, proxyPath=None): """ Get all the users of the VOMS VO with their detailed information + :param str proxyPath: proxy path + :return: user dictionary keyed by the user DN """ if not self.urls: return S_ERROR(DErrno.ENOAUTH, "No VOMS server defined") - userProxy = getProxyLocation() + userProxy = proxyPath or getProxyLocation() caPath = getCAsLocation() rawUserList = [] result = None @@ -125,7 +127,6 @@ def getUsers(self): resultDict[dn] = user resultDict[dn]['CA'] = cert['issuerString'] resultDict[dn]['certSuspended'] = cert.get('suspended') - resultDict[dn]['suspended'] = user.get('suspended') resultDict[dn]['mail'] = user.get('emailAddress') resultDict[dn]['Roles'] = user.get('fqans') attributes = user.get('attributes') diff --git a/src/DIRAC/Core/Tornado/Server/BaseRequestHandler.py b/src/DIRAC/Core/Tornado/Server/BaseRequestHandler.py new file mode 100644 index 00000000000..eb4c99920c9 --- /dev/null +++ b/src/DIRAC/Core/Tornado/Server/BaseRequestHandler.py @@ -0,0 +1,704 @@ +""" BaseRequestHandler is the base class for tornados services and etc handlers. + It directly inherits from :py:class:`tornado.web.RequestHandler` +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from io import open + +import os +import time +import threading +from datetime import datetime +from six.moves import http_client +from six.moves.urllib.parse import unquote + +import tornado +from tornado import gen +from tornado.web import RequestHandler, HTTPError +from tornado.ioloop import IOLoop +from tornado.httpclient import HTTPResponse +from tornado.concurrent import Future + +import DIRAC + +from DIRAC import gConfig, gLogger, S_OK, S_ERROR +from DIRAC.Core.DISET.AuthManager import AuthManager +from DIRAC.Core.Utilities.JEncode import decode, encode +from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error +from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient + +sLog = gLogger.getSubLogger(__name__) + + +class BaseRequestHandler(RequestHandler): + # Because we initialize at first request, we use a flag to know if it's already done + __init_done = False + # Lock to make sure that two threads are not initializing at the same time + __init_lock = threading.RLock() + + # MonitoringClient, we don't use gMonitor which is not thread-safe + # We also need to add specific attributes for each service + _monitor = None + + # Auth requirements + AUTH_PROPS = None + # Type of component + MONITORING_COMPONENT = MonitoringClient.COMPONENT_WEB + # Authentication types + AUTHZ_GRANTS = ['SSL', 'JWT'] + # Prefix of methods names + METHOD_PREFIX = "export_" + + @classmethod + def _initMonitoring(cls, serviceName, fullUrl): + """ + Initialize the monitoring specific to this handler + This has to be called only by :py:meth:`.__initializeService` + to ensure thread safety and unicity of the call. + + :param serviceName: relative URL ``//`` + :param fullUrl: full URl like ``https://://`` + """ + + # Init extra bits of monitoring + + cls._monitor = MonitoringClient() + cls._monitor.setComponentType(cls.MONITORING_COMPONENT) + + cls._monitor.initialize() + + if tornado.process.task_id() is None: # Single process mode + cls._monitor.setComponentName('Tornado/%s' % serviceName) + else: + cls._monitor.setComponentName('Tornado/CPU%d/%s' % (tornado.process.task_id(), serviceName)) + + cls._monitor.setComponentLocation(fullUrl) + + cls._monitor.registerActivity("Queries", "Queries served", "Framework", "queries", MonitoringClient.OP_RATE) + + cls._monitor.setComponentExtraParam('DIRACVersion', DIRAC.version) + cls._monitor.setComponentExtraParam('platform', DIRAC.getPlatform()) + cls._monitor.setComponentExtraParam('startTime', datetime.utcnow()) + + cls._stats = {'requests': 0, 'monitorLastStatsUpdate': time.time()} + + return S_OK() + + @classmethod + def _getServiceName(cls, request): + """ Search service name in request. + + :param object request: tornado Request + + :return: str + """ + raise NotImplementedError() + + @classmethod + def _getServiceAuthSection(cls, serviceName): + """ Search service auth section. + + :param str serviceName: service name + + :return: str + """ + return "%s/Authorization" % PathFinder.getServiceSection(serviceName) + + @classmethod + def _getServiceInfo(cls, serviceName, request): + """ Fill service information. + + :param str serviceName: service name + :param object request: tornado Request + + :return: dict + """ + return {} + + @classmethod + def __initializeService(cls, request): + """ + Initialize a service. + The work is only perform once at the first request. + + :param object request: tornado Request + + :returns: S_OK + """ + # If the initialization was already done successfuly, + # we can just return + if cls.__init_done: + return S_OK() + + # Otherwise, do the work but with a lock + with cls.__init_lock: + + # Check again that the initialization was not done by another thread + # while we were waiting for the lock + if cls.__init_done: + return S_OK() + + # absoluteUrl: full URL e.g. ``https://://`` + absoluteUrl = request.path + serviceName = cls._getServiceName(request) + + cls._startTime = datetime.utcnow() + sLog.info("First use of %s, initializing service..." % serviceName) + cls._authManager = AuthManager(cls._getServiceAuthSection(serviceName)) + + cls._initMonitoring(serviceName, absoluteUrl) + + cls._serviceName = serviceName + cls._validNames = [serviceName] + serviceInfo = cls._getServiceInfo(serviceName, request) + + cls._serviceInfoDict = serviceInfo + + cls.__monitorLastStatsUpdate = time.time() + + cls.initializeHandler(serviceInfo) + + cls.__init_done = True + + return S_OK() + + @classmethod + def initializeHandler(cls, serviceInfo): + """ + This may be overwritten when you write a DIRAC service handler + And it must be a class method. This method is called only one time, + at the first request + + :param dict ServiceInfoDict: infos about services, it contains + 'serviceName', 'serviceSectionPath', + 'csPaths' and 'URL' + """ + pass + + def initializeRequest(self): + """ + Called at every request, may be overwritten in your handler. + """ + pass + + # This is a Tornado magic method + def initialize(self): # pylint: disable=arguments-differ + """ + Initialize the handler, called at every request. + + It just calls :py:meth:`.__initializeService` + + If anything goes wrong, the client will get ``Connection aborted`` + error. See details inside the method. + + ..warning:: + DO NOT REWRITE THIS FUNCTION IN YOUR HANDLER + ==> initialize in DISET became initializeRequest in HTTPS ! + """ + # Only initialized once + if not self.__init_done: + # Ideally, if something goes wrong, we would like to return a Server Error 500 + # but this method cannot write back to the client as per the + # `tornado doc `_. + # So the client will get a ``Connection aborted``` + try: + res = self.__initializeService(self.request) + if not res['OK']: + raise Exception(res['Message']) + except Exception as e: + sLog.error("Error in initialization", repr(e)) + raise + + def _monitorRequest(self): + """ Monitor action for each request + """ + self._stats['requests'] += 1 + self._monitor.setComponentExtraParam('queries', self._stats['requests']) + self._monitor.addMark("Queries") + + def _getMethodName(self): + """ Parse method name. + + :return: str + """ + raise NotImplementedError() + + def _getMethodArgs(self, args): + """ Decode args. + + :return: list + """ + return args + + def _getMethodAuthProps(self): + """ Resolves the hard coded authorization requirements for method. + + :return: object + """ + try: + return getattr(self, 'auth_' + self.method) + except AttributeError: + if not isinstance(self.AUTH_PROPS, (list, tuple)): + self.AUTH_PROPS = [p.strip() for p in self.AUTH_PROPS.split(",") if p.strip()] + return self.AUTH_PROPS + + def _getMethod(self): + """ Get method object. + + :return: object + """ + try: + return getattr(self, '%s%s' % (self.METHOD_PREFIX, self.method)) + except AttributeError as e: + sLog.error("Invalid method", self.method) + raise HTTPError(status_code=http_client.NOT_IMPLEMENTED) + + def prepare(self): + """ + Prepare the request. It reads certificates and check authorizations. + We make the assumption that there is always going to be a ``method`` argument + regardless of the HTTP method used + + """ + + # "method" argument of the POST call. + # This resolves into the ``export_`` method + # on the handler side + # If the argument is not available, the method exists + # and an error 400 ``Bad Request`` is returned to the client + self.method = self._getMethodName() + + self._monitorRequest() + + try: + self.credDict = self._gatherPeerCredentials() + except Exception as e: # pylint: disable=broad-except + # If an error occur when reading certificates we close connection + # It can be strange but the RFC, for HTTP, say's that when error happend + # before authentication we return 401 UNAUTHORIZED instead of 403 FORBIDDEN + sLog.debug(str(e)) + sLog.error( + "Error gathering credentials ", "%s; path %s" % + (self.getRemoteAddress(), self.request.path)) + raise HTTPError(status_code=http_client.UNAUTHORIZED) + + # Check whether we are authorized to perform the query + # Note that performing the authQuery modifies the credDict... + authorized = self._authManager.authQuery(self.method, self.credDict, + self._getMethodAuthProps()) + if not authorized: + extraInfo = '' + if self.credDict.get('DN'): + extraInfo += 'DN: %s' % self.credDict['DN'] + if self.credDict.get('ID'): + extraInfo += 'ID: %s' % self.credDict['ID'] + sLog.error( + "Unauthorized access", "Identity %s; path %s; %s" % + (self.srv_getFormattedRemoteCredentials(), + self.request.path, extraInfo)) + raise HTTPError(status_code=http_client.UNAUTHORIZED) + + # Make post a coroutine. + # See https://www.tornadoweb.org/en/branch5.1/guide/coroutines.html#coroutines + # for details + @gen.coroutine + def post(self, *args, **kwargs): # pylint: disable=arguments-differ + """ + Method to handle incoming ``POST`` requests. + Note that all the arguments are already prepared in the :py:meth:`.prepare` + method. + + The ``POST`` arguments expected are: + + * ``method``: name of the method to call + * ``args``: JSON encoded arguments for the method + * ``extraCredentials``: (optional) Extra informations to authenticate client + * ``rawContent``: (optionnal, default False) If set to True, return the raw output + of the method called. + + If ``rawContent`` was requested by the client, the ``Content-Type`` + is ``application/octet-stream``, otherwise we set it to ``application/json`` + and JEncode retVal. + + If ``retVal`` is a dictionary that contains a ``Callstack`` item, + it is removed, not to leak internal information. + + + Example of call using ``requests``:: + + In [20]: url = 'https://server:8443/DataManagement/TornadoFileCatalog' + ...: cert = '/tmp/x509up_u1000' + ...: kwargs = {'method':'whoami'} + ...: caPath = '/home/dirac/ClientInstallDIR/etc/grid-security/certificates/' + ...: with requests.post(url, data=kwargs, cert=cert, verify=caPath) as r: + ...: print r.json() + ...: + {u'OK': True, + u'Value': {u'DN': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'group': u'dirac_user', + u'identity': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'isLimitedProxy': False, + u'isProxy': True, + u'issuer': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'properties': [u'NormalUser'], + u'secondsLeft': 85441, + u'subject': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch/CN=2409820262', + u'username': u'adminusername', + u'validDN': False, + u'validGroup': False}} + """ + # Execute the method in an executor (basically a separate thread) + # Because of that, we cannot calls certain methods like `self.write` + # in _executeMethod. This is because these methods are not threadsafe + # https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes + # However, we can still rely on instance attributes to store what should + # be sent back (reminder: there is an instance + # of this class created for each request) + retVal = yield IOLoop.current().run_in_executor(None, self._executeMethod, args) + + # retVal is :py:class:`tornado.concurrent.Future` + self._finishFuture(retVal) + + @gen.coroutine + def _executeMethod(self, args): + """ + Execute the method called, this method is ran in an executor + We have several try except to catch the different problem which can occur + + - First, the method does not exist => Attribute error, return an error to client + - second, anything happend during execution => General Exception, send error to client + + .. warning:: + This method is called in an executor, and so cannot use methods like self.write + See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes + """ + + sLog.notice( + "Incoming request %s /%s: %s" % + (self.srv_getFormattedRemoteCredentials(), + self._serviceName, + self.method)) + + # getting method + method = self._getMethod() + methodArgs = self._getMethodArgs(args) + + # Execute + try: + self.initializeRequest() + retVal = method(*args) + except Exception as e: # pylint: disable=broad-except + sLog.exception("Exception serving request", "%s:%s" % (str(e), repr(e))) + raise HTTPError(http_client.INTERNAL_SERVER_ERROR) + + return retVal + + def _finishFuture(self, retVal): + """ Handler Future result + + :param object retVal: tornado.concurrent.Future + """ + + # Wait result only if it's a Future object + self.result = retVal.result() if isinstance(retVal, Future) else retVal + + # Here it is safe to write back to the client, because we are not + # in a thread anymore + + # Parse HTTPResponse + if isinstance(self.result, HTTPResponse): + self.set_status(self.result.code) + for key in self.result.headers: + self.set_header(key, self.result.headers[key]) + if self.result.body: + self.write(self.result.body) + + # If set to true, do not JEncode the return of the RPC call + # This is basically only used for file download through + # the 'streamToClient' method. + elif self.get_argument('rawContent', default=False): + # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt + self.set_header("Content-Type", "application/octet-stream") + self.write(self.result) + + # Return simple text or html + elif isinstance(self.result, str): + self.write(self.result) + + # DIRAC JSON + else: + self.set_header("Content-Type", "application/json") + self.write(encode(self.result)) + + self.finish() + + def on_finish(self): + """ + Called after the end of HTTP request. + Log the request duration + """ + elapsedTime = 1000.0 * self.request.request_time() + + argsString = "OK" + try: + if not self.result['OK']: + argsString = "ERROR: %s" % self.result['Message'] + except (AttributeError, KeyError, TypeError): # In case it is not a DIRAC structure + if self._reason != 'OK': + argsString = 'ERROR %s' % self._reason + + sLog.notice("Returning response", "%s %s (%.2f ms) %s" % (self.srv_getFormattedRemoteCredentials(), + self._serviceName, + elapsedTime, argsString)) + + def _gatherPeerCredentials(self): + """ Returne a dictionary designed to work with the AuthManager, + already written for DISET and re-used for HTTPS. + + :returns: a dict containing the return of :py:meth:`DIRAC.Core.Security.X509Chain.X509Chain.getCredentials` + (not a DIRAC structure !) + """ + err = '' + result = S_OK({}) + for a in self.AUTHZ_GRANTS: + if a.upper() == 'SSL': + result = self.__authzCertificate() + elif a.upper() == 'JWT': + result = self.__authzToken() + else: + raise Exception('%s authentication type is not supported.' % a) + if result['OK']: + break + err += '%s authentication: %s; ' + if err: + raise Exception(err) + return result['Value'] + + def __authzCertificate(self): + """ Load client certchain in DIRAC and extract informations. + + :return: S_OK(dict)/S_ERROR() + """ + peerChain = X509Chain() + derCert = self.request.get_ssl_certificate() + + # Get client certificate pem + if derCert: + chainAsText = derCert.as_pem() + # Here we read all certificate chain + cert_chain = self.request.get_ssl_certificate_chain() + for cert in cert_chain: + chainAsText += cert.as_pem() + elif self.request.headers.get('X-Ssl_client_verify') == 'SUCCESS': + chainAsTextEncoded = self.request.headers.get('X-SSL-CERT') + chainAsText = unquote(chainAsTextEncoded) + else: + return S_ERROR('Not found a valide client certificate.') + + peerChain.loadChainFromString(chainAsText) + + # Retrieve the credentials + res = peerChain.getCredentials(withRegistryInfo=False) + if not res['OK']: + return res + + credDict = res['Value'] + + # We check if client sends extra credentials... + if "extraCredentials" in self.request.arguments: + extraCred = self.get_argument("extraCredentials") + if extraCred: + credDict['extraCredentials'] = decode(extraCred)[0] + return S_OK(credDict) + + def __authzToken(self): + """ Load token claims in DIRAC and extract informations. + + :return: S_OK(dict)/S_ERROR() + """ + try: + token = ResourceProtector().acquire_token(self.request, scope) + except Exception as e: + return S_ERROR(str(e)) + return {'ID': token.sub, 'issuer': token.issuer, 'group': token.groups[0]} + + @property + def log(self): + return sLog + + def getDN(self): + return self.credDict.get('DN', '') + + def getID(self): + return self.credDict.get('ID', '') + + def getUserName(self): + return self.credDict.get('username', '') + + def getUserGroup(self): + return self.credDict.get('group', '') + + def getProperties(self): + return self.credDict.get('properties', []) + + def isRegisteredUser(self): + return self.credDict.get('username', 'anonymous') != 'anonymous' and self.credDict.get('group') + + auth_ping = ['all'] + + def export_ping(self): + """ + Default ping method, returns some info about server. + + It returns the exact same information as DISET, for transparency purpose. + """ + # COPY FROM DIRAC.Core.DISET.RequestHandler + dInfo = {} + dInfo['version'] = DIRAC.version + dInfo['time'] = datetime.utcnow() + # Uptime + try: + with open("/proc/uptime", 'rt') as oFD: + iUptime = int(float(oFD.readline().split()[0].strip())) + dInfo['host uptime'] = iUptime + except Exception: # pylint: disable=broad-except + pass + startTime = self._startTime + dInfo['service start time'] = self._startTime + serviceUptime = datetime.utcnow() - startTime + dInfo['service uptime'] = serviceUptime.days * 3600 + serviceUptime.seconds + # Load average + try: + with open("/proc/loadavg", 'rt') as oFD: + dInfo['load'] = " ".join(oFD.read().split()[:3]) + except Exception: # pylint: disable=broad-except + pass + dInfo['name'] = self._serviceInfoDict['serviceName'] + stTimes = os.times() + dInfo['cpu times'] = {'user time': stTimes[0], + 'system time': stTimes[1], + 'children user time': stTimes[2], + 'children system time': stTimes[3], + 'elapsed real time': stTimes[4] + } + + return S_OK(dInfo) + + auth_echo = ['all'] + + @staticmethod + def export_echo(data): + """ + This method used for testing the performance of a service + """ + return S_OK(data) + + auth_whoami = ['authenticated'] + + def export_whoami(self): + """ + A simple whoami, returns all credential dictionary, except certificate chain object. + """ + credDict = self.srv_getRemoteCredentials() + if 'x509Chain' in credDict: + # Not serializable + del credDict['x509Chain'] + return S_OK(credDict) + + @classmethod + def srv_getCSOption(cls, optionName, defaultValue=False): + """ + Get an option from the CS section of the services + + :return: Value for serviceSection/optionName in the CS being defaultValue the default + """ + if optionName[0] == "/": + return gConfig.getValue(optionName, defaultValue) + for csPath in cls._serviceInfoDict['csPaths']: + result = gConfig.getOption("%s/%s" % (csPath, optionName, ), defaultValue) + if result['OK']: + return result['Value'] + return defaultValue + + def getCSOption(self, optionName, defaultValue=False): + """ + Just for keeping same public interface + """ + return self.srv_getCSOption(optionName, defaultValue) + + def srv_getRemoteAddress(self): + """ + Get the address of the remote peer. + + :return: Address of remote peer. + """ + + remote_ip = self.request.remote_ip + # Although it would be trivial to add this attribute in _HTTPRequestContext, + # Tornado won't release anymore 5.1 series, so go the hacky way + try: + remote_port = self.request.connection.stream.socket.getpeername()[1] + except Exception: # pylint: disable=broad-except + remote_port = 0 + + return (remote_ip, remote_port) + + def getRemoteAddress(self): + """ + Just for keeping same public interface + """ + return self.srv_getRemoteAddress() + + def srv_getRemoteCredentials(self): + """ + Get the credentials of the remote peer. + + :return: Credentials dictionary of remote peer. + """ + return self.credDict + + def getRemoteCredentials(self): + """ + Get the credentials of the remote peer. + + :return: Credentials dictionary of remote peer. + """ + return self.credDict + + def srv_getFormattedRemoteCredentials(self): + """ + Return the DN of user + + Mostly copy paste from + :py:meth:`DIRAC.Core.DISET.private.Transports.BaseTransport.BaseTransport.getFormattedCredentials` + + Note that the information will be complete only once the AuthManager was called + """ + address = self.getRemoteAddress() + peerId = "" + # Depending on where this is call, it may be that credDict is not yet filled. + # (reminder: AuthQuery fills part of it..) + try: + peerId = "[%s:%s]" % (self.credDict['group'], self.credDict['username']) + except AttributeError: + pass + + if address[0].find(":") > -1: + return "([%s]:%s)%s" % (address[0], address[1], peerId) + return "(%s:%s)%s" % (address[0], address[1], peerId) + + def srv_getServiceName(self): + """ + Return the service name + """ + return self._serviceInfoDict['serviceName'] + + def srv_getURL(self): + """ + Return the URL + """ + return self.request.path diff --git a/src/DIRAC/Core/Tornado/Server/HandlerManager.py b/src/DIRAC/Core/Tornado/Server/HandlerManager.py index 1f4397f63e9..5d420357b48 100644 --- a/src/DIRAC/Core/Tornado/Server/HandlerManager.py +++ b/src/DIRAC/Core/Tornado/Server/HandlerManager.py @@ -9,6 +9,8 @@ __RCSID__ = "$Id$" +from six import string_types +import inspect from tornado.web import url as TornadoURL, RequestHandler from DIRAC import gConfig, gLogger, S_ERROR, S_OK @@ -50,109 +52,219 @@ class HandlerManager(object): ``System/Component`` (e.g. ``DataManagement/FileCatalog``) """ - def __init__(self, autoDiscovery=True): + def __init__(self, services=True, endpoints=False): """ Initialization function, you can set autoDiscovery=False to prevent automatic discovery of handler. If disabled you can use loadHandlersByServiceName() to load your handlers or loadHandlerInHandlerManager() - :param autoDiscovery: (default True) Disable the automatic discovery, - can be used to choose service we want to load. + :param services: (default True) List of service handlers to load. + If ``True``, loads all services from CS + :type services: bool or list + :param endpoints: (default False) List of endpoint handlers to load. + If ``True``, loads all endpoints from CS + :type endpoints: bool or list """ + self.loader = None self.__handlers = {} + self.__services = services + self.__endpoints = endpoints self.__objectLoader = ObjectLoader() - self.__autoDiscovery = autoDiscovery - self.loader = ModuleLoader("Service", PathFinder.getServiceSection, RequestHandler, moduleSuffix="Handler") - def __addHandler(self, handlerTuple, url=None): + def __addHandler(self, handlerPath, handler, urls=None, port=None): """ Function which add handler to list of known handlers + :param str handlerPath: module name, e.g.: `Framework/Auth` + :param object handler: handler class + :param list urls: request path + :param int port: port - :param handlerTuple: (path, class) + :return: S_OK()/S_ERROR() """ - # Check if handler not already loaded - if not url or url not in self.__handlers: - gLogger.debug("Find new handler %s" % (handlerTuple[0])) - - # If url is not given, try to discover it - if url is None: - # FIRST TRY: Url is hardcoded - try: - url = handlerTuple[1].LOCATION - # SECOND TRY: URL can be deduced from path - except AttributeError: - gLogger.debug("No location defined for %s try to get it from path" % handlerTuple[0]) - url = urlFinder(handlerTuple[0]) - + # First of all check if we can find route + # If urls is not given, try to discover it + if urls is None: + # FIRST TRY: Url is hardcoded + try: + urls = handler.LOCATION + # SECOND TRY: URL can be deduced from path + except AttributeError: + gLogger.debug("No location defined for %s try to get it from path" % handlerPath) + urls = urlFinder(handlerPath) + + if not urls: + gLogger.warn("URL not found for %s" % (handlerPath)) + return S_ERROR("URL not found for %s" % (handlerPath)) + + for url in urls if isinstance(urls, (list, tuple)) else [urls]: # We add "/" if missing at begin, e.g. we found "Framework/Service" # URL can't be relative in Tornado if url and not url.startswith('/'): url = "/%s" % url - elif not url: - gLogger.warn("URL not found for %s" % (handlerTuple[0])) - return S_ERROR("URL not found for %s" % (handlerTuple[0])) + + # Some new handler + if handlerPath not in self.__handlers: + gLogger.debug("Add new handler %s with port %s" % (handlerPath, port)) + self.__handlers[handlerPath] = {'URLs': [], 'Port': port} + + # Check if URL already loaded + if (url, handler) in self.__handlers[handlerPath]['URLs']: + gLogger.debug("URL: %s already loaded for %s " % (url, handlerPath)) + continue # Finally add the URL to handlers - if url not in self.__handlers: - self.__handlers[url] = handlerTuple[1] - gLogger.info("New handler: %s with URL %s" % (handlerTuple[0], url)) - else: - gLogger.debug("Handler already loaded %s" % (handlerTuple[0])) + gLogger.info("Add new URL %s to %s handler" % (url, handlerPath)) + self.__handlers[handlerPath]['URLs'].append((url, handler)) + return S_OK() - def discoverHandlers(self): + def discoverHandlers(self, handlerInstance): """ Force the discovery of URL, automatic call when we try to get handlers for the first time. You can disable the automatic call with autoDiscovery=False at initialization + + :param str handlerInstance: handler instance, the name of the section in some system section e.g.:: Services, APIs + + :return: list """ - gLogger.debug("Trying to auto-discover the handlers for Tornado") + urls = [] + gLogger.debug("Trying to auto-discover the %s handlers for Tornado" % handlerInstance) # Look in config diracSystems = gConfig.getSections('/Systems') - serviceList = [] if diracSystems['OK']: for system in diracSystems['Value']: try: - instance = PathFinder.getSystemInstance(system) - services = gConfig.getSections('/Systems/%s/%s/Services' % (system, instance)) - if services['OK']: - for service in services['Value']: - newservice = ("%s/%s" % (system, service)) - - # We search in the CS all handlers which used HTTPS as protocol - isHTTPS = gConfig.getValue('/Systems/%s/%s/Services/%s/Protocol' % (system, instance, service)) - if isHTTPS and isHTTPS.lower() == 'https': - serviceList.append(newservice) + sysInstance = PathFinder.getSystemInstance(system) + result = gConfig.getSections('/Systems/%s/%s/%s' % (system, sysInstance, handlerInstance)) + if result['OK']: + for inst in result['Value']: + newInst = ("%s/%s" % (system, inst)) + + if handlerInstance == 'Services': + # We search in the CS all handlers which used HTTPS as protocol + isHTTPS = gConfig.getValue('/Systems/%s/%s/Services/%s/Protocol' % (system, sysInstance, inst)) + if isHTTPS and isHTTPS.lower() == 'https': + urls.append(newInst) + else: + port = gConfig.getValue('/Systems/%s/%s/Services/%s/Port' % (system, sysInstance, inst)) + if port: + newInst += ':%s' % port + urls.append(newInst) # On systems sometime you have things not related to services... except RuntimeError: pass - return self.loadHandlersByServiceName(serviceList) + return urls - def loadHandlersByServiceName(self, servicesNames): + def loadServicesHandlers(self, services=None): """ Load a list of handler from list of service using DIRAC moduleLoader Use :py:class:`DIRAC.Core.Base.private.ModuleLoader` - :param servicesNames: list of service, e.g. ['Framework/Hello', 'Configuration/Server'] + :param services: List of service handlers to load. Default value set at initialization + If ``True``, loads all services from CS + :type services: bool or list + + :return: S_OK()/S_ERROR() """ + # list of services, e.g. ['Framework/Hello', 'Configuration/Server'] + if isinstance(services, string_types): + services = [services] + # list of services + self.__services = self.__services if services is None else services if services else [] + + if self.__services == True: + self.__services = self.discoverHandlers('Services') + + if self.__services: + self.loader = ModuleLoader("Service", PathFinder.getServiceSection, RequestHandler, moduleSuffix="Handler") + + # Use DIRAC system to load: search in CS if path is given and if not defined + # it search in place it should be (e.g. in DIRAC/FrameworkSystem/Service) + load = self.loader.loadModules(self.__services) + if not load['OK']: + return load + for module in self.loader.getModules().values(): + url = module['loadName'] + + # URL can be like https://domain:port/service/name or just service/name + # Here we just want the service name, for tornado + serviceTuple = url.replace('https://', '').split('/')[-2:] + url = "%s/%s" % (serviceTuple[0], serviceTuple[1]) + self.__addHandler(module['loadName'], module['classObj'], url) + return S_OK() + + def __extractPorts(self, urls): + """ Extract ports from urls - # Use DIRAC system to load: search in CS if path is given and if not defined - # it search in place it should be (e.g. in DIRAC/FrameworkSystem/Service) - if not isinstance(servicesNames, list): - servicesNames = [servicesNames] + :param list urls: urls that can contain port, .e.g:: System/Service:port - load = self.loader.loadModules(servicesNames) - if not load['OK']: - return load - for module in self.loader.getModules().values(): - url = module['loadName'] + :return: (dict, list) + """ + portMapping = {} + newURLs = [] + for _url in urls: + if ':' in _url: + urlTuple = _url.split(':') + if urlTuple[0] not in portMapping: + portMapping[urlTuple[0]] = urlTuple[1] + newURLs.append(urlTuple[0]) + else: + newURLs.append(_url) + return (portMapping, newURLs) + + def loadEndpointsHandlers(self, endpoints=None): + """ + Load a list of handler from list of endpoints using DIRAC moduleLoader + Use :py:class:`DIRAC.Core.Base.private.ModuleLoader` - # URL can be like https://domain:port/service/name or just service/name - # Here we just want the service name, for tornado - serviceTuple = url.replace('https://', '').split('/')[-2:] - url = "%s/%s" % (serviceTuple[0], serviceTuple[1]) - self.__addHandler((module['loadName'], module['classObj']), url) + :param endpoints: List of endpoint handlers to load. Default value set at initialization + If ``True``, loads all endpoints from CS + :type endpoints: bool or list + + :return: S_OK()/S_ERROR() + """ + # list of endpoints, e.g. ['Framework/Hello', 'Configuration/Conf'] + if isinstance(endpoints, string_types): + endpoints = [endpoints] + # list of endpoints. If __endpoints is ``True`` than list of endpoints will dicover from CS + self.__endpoints = self.__endpoints if endpoints is None else endpoints if endpoints else [] + + if self.__endpoints == True: + self.__endpoints = self.discoverHandlers('APIs') + + if self.__endpoints: + # Extract ports + ports, self.__endpoints = self.__extractPorts(self.__endpoints) + + self.loader = ModuleLoader("API", PathFinder.getAPISection, RequestHandler, moduleSuffix="Handler") + + # Use DIRAC system to load: search in CS if path is given and if not defined + # it search in place it should be (e.g. in DIRAC/FrameworkSystem/API) + load = self.loader.loadModules(self.__endpoints) + if not load['OK']: + return load + for module in self.loader.getModules().values(): + handler = module['classObj'] + if not handler.LOCATION: + handler.LOCATION = urlFinder(module['loadName']) + urls = [] + # Look for methods that are exported + for mName, mObj in inspect.getmembers(handler): + if inspect.ismethod(mObj) and mName.find(handler.METHOD_PREFIX) == 0: + methodName = mName[len(handler.METHOD_PREFIX):] + args = getattr(handler, 'path_%s' % methodName, []) + # argObj = inspect.getargspec(mObj) + # args = '/'.join(['([A-z0-9_.-]+)'] * (len(argObj.args) - 1 - len(argObj.defaults or []))) + # defs = '/'.join(['([A-z0-9_.-]*)'] * len(argObj.defaults or [])) + gLogger.debug(" - Route %s/%s -> %s %s" % (handler.LOCATION, methodName, module['loadName'], mName)) + url = "%s%s" % (handler.LOCATION, '' if methodName == 'index' else ('/%s' % methodName)) + if args: + url += '[\/]?%s' % '/'.join(args) + urls.append(url) + gLogger.debug(" * %s" % url) + self.__addHandler(module['loadName'], handler, urls, ports.get(module['modName'])) return S_OK() def getHandlersURLs(self): @@ -163,24 +275,32 @@ def getHandlersURLs(self): :returns: a list of URL (not the string with "https://..." but the tornado object) see http://www.tornadoweb.org/en/stable/web.html#tornado.web.URLSpec """ - if not self.__handlers and self.__autoDiscovery: - self.__autoDiscovery = False - self.discoverHandlers() urls = [] - for key in self.__handlers: - urls.append(TornadoURL(key, self.__handlers[key])) + for handlerData in self.__handlers.values(): + for url in handlerData['URLs']: + urls.append(TornadoURL(url)) return urls def getHandlersDict(self): """ Return all handler dictionary - :returns: dictionary with absolute url as key ("/System/Service") - and tornado.web.url object as value + :returns: dictionary with service name as key ("System/Service") + and tornado.web.url objects as value for 'URLs' key + and port as value for 'Port' key """ - if not self.__handlers and self.__autoDiscovery: - self.__autoDiscovery = False - res = self.discoverHandlers() - if not res['OK']: - gLogger.error("Could not load handlers", res) return self.__handlers + + def getRoutes(self, defaultPort): + """ Get routes for each of the registred port + + :return list -- contain tuples with port and list of tornado urls + """ + routes = {} + for data in self.__handlers.values(): + port = data.get('Port') + settings = data.get('Settings') + if port not in routes: + routes[port] = [] + routes[port] = list(set(routes[port]) + set(data['URLs'])) + return list(routes.items()) diff --git a/src/DIRAC/Core/Tornado/Server/TornadoREST.py b/src/DIRAC/Core/Tornado/Server/TornadoREST.py new file mode 100644 index 00000000000..563cdff39c6 --- /dev/null +++ b/src/DIRAC/Core/Tornado/Server/TornadoREST.py @@ -0,0 +1,85 @@ +""" +TornadoService is the base class for your handlers. +It directly inherits from :py:class:`tornado.web.RequestHandler` +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import tornado.ioloop +from tornado import gen +from tornado.web import HTTPError +from tornado.ioloop import IOLoop +from six.moves import http_client + +import DIRAC + +from DIRAC import gLogger +from DIRAC.Core.Web import Conf +from DIRAC.Core.Tornado.Server.BaseRequestHandler import BaseRequestHandler +from DIRAC.FrameworkSystem.private.authorization.utils.Tokens import ResourceProtector + +sLog = gLogger.getSubLogger(__name__) + + +class TornadoREST(BaseRequestHandler): # pylint: disable=abstract-method + METHOD_PREFIX = 'web_' + + @classmethod + def _getServiceName(cls, request): + """ Search service name in request. + + :param object request: tornado Request + + :return: str + """ + try: + return cls.LOCATION.split('/')[-1].strip('/') + except expression as identifier: + return cls.__name__ + + @classmethod + def _getServiceAuthSection(cls, serviceName): + """ Search service auth section. + + :param str serviceName: service name + + :return: str + """ + return Conf.getAuthSectionForHandler(serviceName) + + def _getMethodName(self): + """ Parse method name. + + :return: str + """ + try: + return self.request.path.split(self.LOCATION)[1].split('?')[0].strip('/').split('/')[0].strip('/') + except Exception: + return 'index' + + @gen.coroutine + def get(self, *args, **kwargs): # pylint: disable=arguments-differ + """ + """ + retVal = yield IOLoop.current().run_in_executor(None, self._executeMethod, args) + + # retVal is :py:class:`tornado.concurrent.Future` + self._finishFuture(retVal) + + def _finishFuture(self, retVal): + """ Handler Future result + + :param object retVal: tornado.concurrent.Future + """ + result = retVal.result() + try: + if not result['OK']: + raise HTTPError(http_client.INTERNAL_SERVER_ERROR) + result = result['Value'] + except (AttributeError, KeyError, TypeError): + pass + super(TornadoREST, self)._finishFuture(result) diff --git a/src/DIRAC/Core/Tornado/Server/TornadoServer.py b/src/DIRAC/Core/Tornado/Server/TornadoServer.py index 3707cea6673..b7365765db1 100644 --- a/src/DIRAC/Core/Tornado/Server/TornadoServer.py +++ b/src/DIRAC/Core/Tornado/Server/TornadoServer.py @@ -9,9 +9,9 @@ __RCSID__ = "$Id$" +import os import time import datetime -import os import M2Crypto @@ -20,21 +20,36 @@ 'tornado_m2crypto.m2iostream.M2IOStream') # pylint: disable=wrong-import-position from tornado.httpserver import HTTPServer -from tornado.web import Application, url +from tornado.web import Application as _Application, url from tornado.ioloop import IOLoop import tornado.ioloop import DIRAC -from DIRAC import gConfig, gLogger -from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.Core.Security import Locations -from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager from DIRAC.Core.Utilities import MemStat +from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager +from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.Security import Locations, X509Chain, X509CRL from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient +## FROM WEB +import sys +import signal +import tornado.process +import tornado.autoreload + +from DIRAC.FrameworkSystem.private.authorization.utils.Sessions import SessionManager + sLog = gLogger.getSubLogger(__name__) +class Application(_Application, SessionManager): + def __init__(self, *args, **kwargs): + _Application.__init__(self, *args, **kwargs) + SessionManager.__init__(self) + + class TornadoServer(object): """ Tornado webserver @@ -60,35 +75,44 @@ class TornadoServer(object): Example 2:We want to debug service1 and service2 only, and use another port for that :: - services = ['component/service1', 'component/service2'] - serverToLaunch = TornadoServer(services=services, port=1234) + services = ['component/service1:port1', 'component/service2'] + endpoints = ['component/endpoint1:port1', 'component/endpoint2'] + serverToLaunch = TornadoServer(services=services, endpoints=endpoints, port=1234) serverToLaunch.startTornado() """ - def __init__(self, services=None, port=None): + def __init__(self, services=None, endpoints=None, port=None, debug=False, balancer=None, processes=None): + """ C'r + + :param list services: (default None) List of service handlers to load. + If ``None``, loads all described in the CS + :param list endpoints: (default None) List of endpoint handlers to load. + If ``None``, loads all described in the CS + :param int port: Port to listen to. + If ``None``, the port is resolved following the logic described in the class documentation + :param str balancer: if need to use balancer, e.g.:: `nginx` + :param int processes: number of processes or if it's True just use all server CPUs """ - - :param list services: (default None) List of service handlers to load. If ``None``, loads all - :param int port: Port to listen to. If None, the port is resolved following the logic - described in the class documentation - """ - + # Debug + self.debug = debug + # Balancer, like as nginx + self.balancer = balancer + # Multiprocessor mode settings + self.processes = 1 if processes is None else 0 if processes is True else processes + if processes: + raise ImportError('Multiprocessor mode is not supported.') + # Applicatio metadata, routes and settings mapping on the ports + self.__appsSettings = {} + # Default port, if enother is not discover if port is None: port = gConfig.getValue("/Systems/Tornado/%s/Port" % PathFinder.getSystemInstance('Tornado'), 8443) - - if services and not isinstance(services, list): - services = [services] - - # URLs for services. - # Contains Tornado :py:class:`tornado.web.url` object - self.urls = [] - # Other infos self.port = port - self.handlerManager = HandlerManager() + # Handler manager initialization with default settings + self.handlerManager = HandlerManager(services, endpoints) + # Monitoring attributes - self._monitor = MonitoringClient() # temp value for computation, used by the monitoring self.__report = None @@ -97,20 +121,82 @@ def __init__(self, services=None, port=None): self.__monitoringLoopDelay = 60 # In secs # If services are defined, load only these ones (useful for debug purpose or specific services) - if services: - retVal = self.handlerManager.loadHandlersByServiceName(services) - if not retVal['OK']: - sLog.error(retVal['Message']) - raise ImportError("Some services can't be loaded, check the service names and configuration.") - + retVal = self.handlerManager.loadServicesHandlers() + if not retVal['OK']: + sLog.error(retVal['Message']) + raise ImportError("Some services can't be loaded, check the service names and configuration.") + + retVal = self.handlerManager.loadEndpointsHandlers() + if not retVal['OK']: + sLog.error(retVal['Message']) + raise ImportError("Some endpoints can't be loaded, check the endpoint names and configuration.") + + def __calculateAppSettings(self): + """ Calculate application information mapping on the ports + """ # if no service list is given, load services from configuration handlerDict = self.handlerManager.getHandlersDict() - for item in handlerDict.items(): - # handlerDict[key].initializeService(key) - self.urls.append(url(item[0], item[1])) - # If there is no services loaded: - if not self.urls: - raise ImportError("There is no services loaded, please check your configuration") + for data in handlerDict.values(): + port = data.get('Port') or self.port + for hURL in data['URLs']: + if port not in self.__appsSettings: + self.__appsSettings[port] = {'routes': [], 'settings': {}} + if hURL not in self.__appsSettings[port]['routes']: + self.__appsSettings[port]['routes'].append(hURL) + return bool(self.__appsSettings) + + def loadServices(self, services): + """ Load a services + + :param services: List of service handlers to load. Default value set at initialization + If ``True``, loads all services from CS + :type services: bool or list + + :return: S_OK()/S_ERROR() + """ + return self.handlerManager.loadServicesHandlers(services) + + def loadEndpoints(self, endpoints): + """ Load a endpoints + + :param endpoints: List of service handlers to load. Default value set at initialization + If ``True``, loads all endpoints from CS + :type endpoints: bool or list + + :return: S_OK()/S_ERROR() + """ + return self.handlerManager.loadEndpointsHandlers(endpoints) + + def addHandlers(self, routes, settings=None, port=None): + """ Add new routes + + :param list routes: routes + :param dict settings: application settings + :param int port: port + """ + port = port or self.port + if port not in self.__appsSettings: + self.__appsSettings[port] = {'routes': [], 'settings': {}} + if settings: + self.__appsSettings[port]['settings'].update(settings) + for route in routes: + if route not in self.__appsSettings[port]['routes']: + self.__appsSettings[port]['routes'].append(route) + + return S_OK() + + # def stopChildProcesses(self, sig, frame): + # """ + # It is used to properly stop tornado when more than one process is used. + # In principle this is doing the job of runsv.... + + # :param int sig: the signal sent to the process + # :param object frame: execution frame which contains the child processes + # """ + # for child in frame.f_locals.get('children', []): + # gLogger.info("Stopping child processes: %d" % child) + # os.kill(child, signal.SIGTERM) + # sys.exit(0) def startTornado(self): """ @@ -118,17 +204,17 @@ def startTornado(self): This method never returns. """ - sLog.debug("Starting Tornado") - self._initMonitoring() + # If there is no services loaded: + if not self.__calculateAppSettings(): + raise Exception("There is no services loaded, please check your configuration") - router = Application(self.urls, - debug=False, - compress_response=True) + sLog.debug("Starting Tornado") + # Prepare SSL settings certs = Locations.getHostCertificateAndKeyLocation() if certs is False: sLog.fatal("Host certificates not found ! Can't start the Server") - raise ImportError("Unable to load certificates") + raise Exception("Unable to load certificates") ca = Locations.getCAsLocation() ssl_options = { 'certfile': certs[0], @@ -138,23 +224,69 @@ def startTornado(self): 'sslDebug': False, # Set to true if you want to see the TLS debug messages } + if self.balancer: + # Create CAs for balancer + generateRevokedCertsFile() # it is used by nginx.... + # when NGINX is used then the Conf.HTTPS return False, it means tornado + # does not have to be configured using 443 port + generateCAFile() # if we use Nginx we have to generate the cas as well... + + # ############ + # # please do no move this lines. The lines must be before the fork_processes + # signal.signal(signal.SIGTERM, self.stopChildProcesses) + # signal.signal(signal.SIGINT, self.stopChildProcesses) + + # # Check processes if we're under a load balancert and have only one port + # if self.processes != 1: + # if not self.balancer: + # raise Exception("For multi processor mode, please, use balacer.") + # if len(self.__appsSettings) != 1: + # raise Exception("For multi processor mode, please, use one server port.") + # tornado.process.fork_processes(self.processes, max_restarts=0) + # ############# + + # Init monitoring + self._initMonitoring() self.__monitorLastStatsUpdate = time.time() self.__report = self.__startReportToMonitoringLoop() - + # Starting monitoring, IOLoop waiting time in ms, __monitoringLoopDelay is defined in seconds tornado.ioloop.PeriodicCallback(self.__reportToMonitoring, self.__monitoringLoopDelay * 1000).start() - # Start server - server = HTTPServer(router, ssl_options=ssl_options, decompress_request=True) - try: - server.listen(self.port) - except Exception as e: # pylint: disable=broad-except - sLog.exception("Exception starting HTTPServer", e) - raise - sLog.always("Listening on port %s" % self.port) - for service in self.urls: - sLog.debug("Available service: %s" % service) - + for port, app in self.__appsSettings.items(): + sLog.debug(" - %s" % "\n - ".join(["%s = %s" % (k, ssl_options[k]) for k in ssl_options])) + + # Default server configuration + settings = dict(debug=self.debug, + compress_response=True, + # Use gLogger instead tornado log + log_function=_logRequest) + + # Merge appllication settings + settings.update(app['settings']) + + # # Don't use autoreload in debug mode for multiprocess + # if self.processes != 1: + # if self.balancer: + # port = 8000 + # port += tornado.process.task_id() or 0 + # settings['debug'] = False + + # Start server + router = Application(app['routes'], **settings) + server = HTTPServer(router, ssl_options=ssl_options, decompress_request=True, xheaders=True) + try: + server.listen(port) + except Exception as e: # pylint: disable=broad-except + sLog.exception("Exception starting HTTPServer", e) + raise + if settings['debug']: + sLog.info("Configuring in developer mode...") + sLog.always("Listening on 127.0.0.1:%s" % port) + for service in app['routes']: + sLog.debug("Available service: %s" % service if isinstance(service, url) else service[0]) + + tornado.autoreload.add_reload_hook(lambda: sLog.verbose("\n == Reloading server...\n")) IOLoop.current().start() def _initMonitoring(self): @@ -224,3 +356,92 @@ def __endReportToMonitoringLoop(self, initialWallTime, initialCPUTime): percentage = cpuTime / wallTime * 100. if percentage > 0: self._monitor.addMark('CPU', percentage) + + +def _logRequest(handler): + """ This function will be called at the end of every request to log the result + + :param object handler: RequestHandler object + """ + print('=== _logRequest') + print('=== %s' % handler) + print(sLog) + print('===============') + status = handler.get_status() + if status < 400: + logm = sLog.notice + elif status < 500: + logm = sLog.warn + else: + logm = sLog.error + request_time = 1000.0 * handler.request.request_time() + logm("%d %s %.2fms" % (status, handler._request_summary(), request_time)) + + +def generateCAFile(): + """ Generate a single CA file with all the PEMs + + :return: str or bool + """ + cert = Locations.getHostCertificateAndKeyLocation() + if cert: + cert = cert[0] + else: + cert = "/opt/dirac/etc/grid-security/hostcert.pem" + + caDir = Locations.getCAsLocation() + for fn in (os.path.join(os.path.dirname(caDir), "cas.pem"), + os.path.join(os.path.dirname(cert), "cas.pem"), + False): + if not fn: + fn = tempfile.mkstemp(prefix="cas.", suffix=".pem")[1] + try: + fd = open(fn, "w") + except IOError: + continue + for caFile in os.listdir(caDir): + caFile = os.path.join(caDir, caFile) + chain = X509Chain.X509Chain() + result = chain.loadChainFromFile(caFile) + if not result['OK']: + continue + expired = chain.hasExpired() + if not expired['OK'] or expired['Value']: + continue + fd.write(chain.dumpAllToString()['Value']) + fd.close() + return fn + return False + + +def generateRevokedCertsFile(): + """ Generate a single CA file with all the PEMs + + :return: str or bool + """ + cert = Locations.getHostCertificateAndKeyLocation() + if cert: + cert = cert[0] + else: + cert = "/opt/dirac/etc/grid-security/hostcert.pem" + + caDir = Locations.getCAsLocation() + for fn in (os.path.join(os.path.dirname(caDir), "allRevokedCerts.pem"), + os.path.join(os.path.dirname(cert), "allRevokedCerts.pem"), + False): + if not fn: + fn = tempfile.mkstemp(prefix="allRevokedCerts", suffix=".pem")[1] + try: + fd = open(fn, "w") + except IOError: + continue + for caFile in os.listdir(caDir): + caFile = os.path.join(caDir, caFile) + chain = X509CRL.X509CRL() + result = chain.loadCRLFromFile(caFile) + if not result['OK']: + continue + fd.write(chain.dumpAllToString()['Value']) + fd.close() + return fn + return False \ No newline at end of file diff --git a/src/DIRAC/Core/Tornado/Server/TornadoService.py b/src/DIRAC/Core/Tornado/Server/TornadoService.py index dbc88d67b28..30ab4228658 100644 --- a/src/DIRAC/Core/Tornado/Server/TornadoService.py +++ b/src/DIRAC/Core/Tornado/Server/TornadoService.py @@ -16,18 +16,20 @@ import threading from datetime import datetime from six.moves import http_client -from tornado.web import RequestHandler, HTTPError -from tornado import gen + +import tornado import tornado.ioloop +from tornado import gen +from tornado.web import RequestHandler, HTTPError from tornado.ioloop import IOLoop import DIRAC from DIRAC import gConfig, gLogger, S_OK, S_ERROR -from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.Core.DISET.AuthManager import AuthManager -from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Core.Utilities.JEncode import decode, encode +from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error +from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient sLog = gLogger.getSubLogger(__name__) @@ -111,625 +113,45 @@ def export_streamToClient(self, myDataToSend, token): These are initialized in the :py:meth:`.initialize` method. """ - - # Because we initialize at first request, we use a flag to know if it's already done - __init_done = False - # Lock to make sure that two threads are not initializing at the same time - __init_lock = threading.RLock() - - # MonitoringClient, we don't use gMonitor which is not thread-safe - # We also need to add specific attributes for each service - _monitor = None + # Prefix of methods names + METHOD_PREFIX = "export_" @classmethod - def _initMonitoring(cls, serviceName, fullUrl): - """ - Initialize the monitoring specific to this handler - This has to be called only by :py:meth:`.__initializeService` - to ensure thread safety and unicity of the call. - - :param serviceName: relative URL ``//`` - :param fullUrl: full URl like ``https://://`` - """ - - # Init extra bits of monitoring + def _getServiceName(cls, request): + """ Search service name in request. - cls._monitor = MonitoringClient() - cls._monitor.setComponentType(MonitoringClient.COMPONENT_WEB) + :param object request: tornado Request - cls._monitor.initialize() - - if tornado.process.task_id() is None: # Single process mode - cls._monitor.setComponentName('Tornado/%s' % serviceName) - else: - cls._monitor.setComponentName('Tornado/CPU%d/%s' % (tornado.process.task_id(), serviceName)) - - cls._monitor.setComponentLocation(fullUrl) - - cls._monitor.registerActivity("Queries", "Queries served", "Framework", "queries", MonitoringClient.OP_RATE) - - cls._monitor.setComponentExtraParam('DIRACVersion', DIRAC.version) - cls._monitor.setComponentExtraParam('platform', DIRAC.getPlatform()) - cls._monitor.setComponentExtraParam('startTime', datetime.utcnow()) - - cls._stats = {'requests': 0, 'monitorLastStatsUpdate': time.time()} - - return S_OK() - - @classmethod - def __initializeService(cls, relativeUrl, absoluteUrl): - """ - Initialize a service. - The work is only perform once at the first request. - - :param relativeUrl: relative URL, e.g. ``//`` - :param absoluteUrl: full URL e.g. ``https://://`` - - :returns: S_OK + :return: str """ - # If the initialization was already done successfuly, - # we can just return - if cls.__init_done: - return S_OK() - - # Otherwise, do the work but with a lock - with cls.__init_lock: - - # Check again that the initialization was not done by another thread - # while we were waiting for the lock - if cls.__init_done: - return S_OK() - - # Url starts with a "/", we just remove it - serviceName = relativeUrl[1:] - - cls._startTime = datetime.utcnow() - sLog.info("First use, initializing service...", "%s" % relativeUrl) - cls._authManager = AuthManager("%s/Authorization" % PathFinder.getServiceSection(serviceName)) - - cls._initMonitoring(serviceName, absoluteUrl) - - cls._serviceName = serviceName - cls._validNames = [serviceName] - serviceInfo = {'serviceName': serviceName, - 'serviceSectionPath': PathFinder.getServiceSection(serviceName), - 'csPaths': [PathFinder.getServiceSection(serviceName)], - 'URL': absoluteUrl - } - cls._serviceInfoDict = serviceInfo - - cls.__monitorLastStatsUpdate = time.time() - - cls.initializeHandler(serviceInfo) - - cls.__init_done = True - - return S_OK() - + # Expected path: ``//`` + return request.path[1:] + @classmethod - def initializeHandler(cls, serviceInfoDict): - """ - This may be overwritten when you write a DIRAC service handler - And it must be a class method. This method is called only one time, - at the first request - - :param dict ServiceInfoDict: infos about services, it contains - 'serviceName', 'serviceSectionPath', - 'csPaths' and 'URL' - """ - pass - - def initializeRequest(self): - """ - Called at every request, may be overwritten in your handler. - """ - pass + def _getServiceInfo(cls, serviceName, request): + """ Fill service information. - # This is a Tornado magic method - def initialize(self): # pylint: disable=arguments-differ - """ - Initialize the handler, called at every request. - - It just calls :py:meth:`.__initializeService` - - If anything goes wrong, the client will get ``Connection aborted`` - error. See details inside the method. - - ..warning:: - DO NOT REWRITE THIS FUNCTION IN YOUR HANDLER - ==> initialize in DISET became initializeRequest in HTTPS ! - """ - - # Only initialized once - if not self.__init_done: - # Ideally, if something goes wrong, we would like to return a Server Error 500 - # but this method cannot write back to the client as per the - # `tornado doc `_. - # So the client will get a ``Connection aborted``` - try: - res = self.__initializeService(self.srv_getURL(), self.request.full_url()) - if not res['OK']: - raise Exception(res['Message']) - except Exception as e: - sLog.error("Error in initialization", repr(e)) - raise - - def prepare(self): - """ - Prepare the request. It reads certificates and check authorizations. - We make the assumption that there is always going to be a ``method`` argument - regardless of the HTTP method used + :param str serviceName: service name + :param object request: tornado Request + :return: dict """ + return {'serviceName': serviceName, + 'serviceSectionPath': PathFinder.getServiceSection(serviceName), + 'csPaths': [PathFinder.getServiceSection(serviceName)], + 'URL': request.full_url()} - # "method" argument of the POST call. - # This resolves into the ``export_`` method - # on the handler side - # If the argument is not available, the method exists - # and an error 400 ``Bad Request`` is returned to the client - self.method = self.get_argument("method") - - self._stats['requests'] += 1 - self._monitor.setComponentExtraParam('queries', self._stats['requests']) - self._monitor.addMark("Queries") - - try: - self.credDict = self._gatherPeerCredentials() - except Exception: # pylint: disable=broad-except - # If an error occur when reading certificates we close connection - # It can be strange but the RFC, for HTTP, say's that when error happend - # before authentication we return 401 UNAUTHORIZED instead of 403 FORBIDDEN - sLog.error( - "Error gathering credentials", "%s; path %s" % - (self.getRemoteAddress(), self.request.path)) - raise HTTPError(status_code=http_client.UNAUTHORIZED) - - # Resolves the hard coded authorization requirements - try: - hardcodedAuth = getattr(self, 'auth_' + self.method) - except AttributeError: - hardcodedAuth = None - - # Check whether we are authorized to perform the query - # Note that performing the authQuery modifies the credDict... - authorized = self._authManager.authQuery(self.method, self.credDict, hardcodedAuth) - if not authorized: - sLog.error( - "Unauthorized access", "Identity %s; path %s; DN %s" % - (self.srv_getFormattedRemoteCredentials, - self.request.path, - self.credDict['DN'], - )) - raise HTTPError(status_code=http_client.UNAUTHORIZED) - - # Make post a coroutine. - # See https://www.tornadoweb.org/en/branch5.1/guide/coroutines.html#coroutines - # for details - @gen.coroutine - def post(self): # pylint: disable=arguments-differ - """ - Method to handle incoming ``POST`` requests. - Note that all the arguments are already prepared in the :py:meth:`.prepare` - method. - - The ``POST`` arguments expected are: - - * ``method``: name of the method to call - * ``args``: JSON encoded arguments for the method - * ``extraCredentials``: (optional) Extra informations to authenticate client - * ``rawContent``: (optionnal, default False) If set to True, return the raw output - of the method called. - - If ``rawContent`` was requested by the client, the ``Content-Type`` - is ``application/octet-stream``, otherwise we set it to ``application/json`` - and JEncode retVal. - - If ``retVal`` is a dictionary that contains a ``Callstack`` item, - it is removed, not to leak internal information. - - - Example of call using ``requests``:: - - In [20]: url = 'https://server:8443/DataManagement/TornadoFileCatalog' - ...: cert = '/tmp/x509up_u1000' - ...: kwargs = {'method':'whoami'} - ...: caPath = '/home/dirac/ClientInstallDIR/etc/grid-security/certificates/' - ...: with requests.post(url, data=kwargs, cert=cert, verify=caPath) as r: - ...: print r.json() - ...: - {u'OK': True, - u'Value': {u'DN': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser', - u'group': u'dirac_user', - u'identity': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser', - u'isLimitedProxy': False, - u'isProxy': True, - u'issuer': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser', - u'properties': [u'NormalUser'], - u'secondsLeft': 85441, - u'subject': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/CN=2409820262', - u'username': u'adminusername', - u'validDN': False, - u'validGroup': False}} - """ + def _getMethodName(self): + """ Parse method name. - sLog.notice( - "Incoming request", "%s /%s: %s" % - (self.srv_getFormattedRemoteCredentials(), - self._serviceName, - self.method)) - - # Execute the method in an executor (basically a separate thread) - # Because of that, we cannot calls certain methods like `self.write` - # in __executeMethod. This is because these methods are not threadsafe - # https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes - # However, we can still rely on instance attributes to store what should - # be sent back (reminder: there is an instance - # of this class created for each request) - retVal = yield IOLoop.current().run_in_executor(None, self.__executeMethod) - - # retVal is :py:class:`tornado.concurrent.Future` - self.result = retVal.result() - - # Here it is safe to write back to the client, because we are not - # in a thread anymore - - # If set to true, do not JEncode the return of the RPC call - # This is basically only used for file download through - # the 'streamToClient' method. - rawContent = self.get_argument('rawContent', default=False) - - if rawContent: - # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt - self.set_header("Content-Type", "application/octet-stream") - result = self.result - else: - self.set_header("Content-Type", "application/json") - result = encode(self.result) - - self.write(result) - self.finish() - - # This nice idea of streaming to the client cannot work because we are ran in an executor - # and we should not write back to the client in a different thread. - # See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes - # def export_streamToClient(self, filename): - # # https://bhch.github.io/posts/2017/12/serving-large-files-with-tornado-safely-without-blocking/ - # #import ipdb; ipdb.set_trace() - # # chunk size to read - # chunk_size = 1024 * 1024 * 1 # 1 MiB - - # with open(filename, 'rb') as f: - # while True: - # chunk = f.read(chunk_size) - # if not chunk: - # break - # try: - # self.write(chunk) # write the chunk to response - # self.flush() # send the chunk to client - # except StreamClosedError: - # # this means the client has closed the connection - # # so break the loop - # break - # finally: - # # deleting the chunk is very important because - # # if many clients are downloading files at the - # # same time, the chunks in memory will keep - # # increasing and will eat up the RAM - # del chunk - # # pause the coroutine so other handlers can run - # yield gen.sleep(0.000000001) # 1 nanosecond - - # return S_OK() - - @gen.coroutine - def __executeMethod(self): + :return: str """ - Execute the method called, this method is ran in an executor - We have several try except to catch the different problem which can occur + return self.get_argument("method") - - First, the method does not exist => Attribute error, return an error to client - - second, anything happend during execution => General Exception, send error to client + def _getMethodArgs(self, args): + """ Decode args. - .. warning:: - This method is called in an executor, and so cannot use methods like self.write - See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes + :return: list """ - - # getting method - try: - # For compatibility reasons with DISET, the methods are still called ``export_*`` - method = getattr(self, 'export_%s' % self.method) - except AttributeError as e: - sLog.error("Invalid method", self.method) - raise HTTPError(status_code=http_client.NOT_IMPLEMENTED) - - # Decode args args_encoded = self.get_body_argument('args', default=encode([])) - - args = decode(args_encoded)[0] - # Execute - try: - self.initializeRequest() - retVal = method(*args) - except Exception as e: # pylint: disable=broad-except - sLog.exception("Exception serving request", "%s:%s" % (str(e), repr(e))) - raise HTTPError(http_client.INTERNAL_SERVER_ERROR) - - return retVal - - # def __write_return(self, retVal): - # """ - # Write back to the client and return. - # It sets some headers (status code, ``Content-Type``). - # If raw content was requested by the client, the ``Content-Type`` - # is ``application/octet-stream``, otherwise we set it to ``application/json`` - # and JEncode retVal. - - # If ``retVal`` is a dictionary that contains a ``Callstack`` item, - # it is removed, not to leak internal information. - - # :param retVal: anything that can be serialized in json. - # """ - - # # In case of error in server side we hide server CallStack to client - # try: - # if 'CallStack' in retVal: - # del retVal['CallStack'] - # except TypeError: - # pass - - # # Set the status - # self.set_status(self._httpStatus) - - # # This is basically only used for file download through - # # the 'streamToClient' method. - # if self.rawContent: - # # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt - # self.set_header("Content-Type", "application/octet-stream") - # returnedData = retVal - # else: - # self.set_header("Content-Type", "application/json") - # returnedData = encode(retVal) - - # self.write(returnedData) - - def on_finish(self): - """ - Called after the end of HTTP request. - Log the request duration - """ - elapsedTime = 1000.0 * self.request.request_time() - - try: - if self.result['OK']: - argsString = "OK" - else: - argsString = "ERROR: %s" % self.result['Message'] - except (AttributeError, KeyError): # In case it is not a DIRAC structure - if self._reason == 'OK': - argsString = 'OK' - else: - argsString = 'ERROR %s' % self._reason - - argsString = "ERROR: %s" % self._reason - sLog.notice("Returning response", "%s %s (%.2f ms) %s" % (self.srv_getFormattedRemoteCredentials(), - self._serviceName, - elapsedTime, argsString)) - - def _gatherPeerCredentials(self): - """ - Load client certchain in DIRAC and extract informations. - - The dictionary returned is designed to work with the AuthManager, - already written for DISET and re-used for HTTPS. - - :returns: a dict containing the return of :py:meth:`DIRAC.Core.Security.X509Chain.X509Chain.getCredentials` - (not a DIRAC structure !) - """ - - chainAsText = self.request.get_ssl_certificate().as_pem() - peerChain = X509Chain() - - # Here we read all certificate chain - cert_chain = self.request.get_ssl_certificate_chain() - for cert in cert_chain: - chainAsText += cert.as_pem() - - peerChain.loadChainFromString(chainAsText) - - # Retrieve the credentials - res = peerChain.getCredentials(withRegistryInfo=False) - if not res['OK']: - raise Exception(res['Message']) - - credDict = res['Value'] - - # We check if client sends extra credentials... - if "extraCredentials" in self.request.arguments: - extraCred = self.get_argument("extraCredentials") - if extraCred: - credDict['extraCredentials'] = decode(extraCred)[0] - return credDict - - -#### -# -# Default method -# -#### - - auth_ping = ['all'] - - def export_ping(self): - """ - Default ping method, returns some info about server. - - It returns the exact same information as DISET, for transparency purpose. - """ - # COPY FROM DIRAC.Core.DISET.RequestHandler - dInfo = {} - dInfo['version'] = DIRAC.version - dInfo['time'] = datetime.utcnow() - # Uptime - try: - with open("/proc/uptime", 'rt') as oFD: - iUptime = int(float(oFD.readline().split()[0].strip())) - dInfo['host uptime'] = iUptime - except Exception: # pylint: disable=broad-except - pass - startTime = self._startTime - dInfo['service start time'] = self._startTime - serviceUptime = datetime.utcnow() - startTime - dInfo['service uptime'] = serviceUptime.days * 3600 + serviceUptime.seconds - # Load average - try: - with open("/proc/loadavg", 'rt') as oFD: - dInfo['load'] = " ".join(oFD.read().split()[:3]) - except Exception: # pylint: disable=broad-except - pass - dInfo['name'] = self._serviceInfoDict['serviceName'] - stTimes = os.times() - dInfo['cpu times'] = {'user time': stTimes[0], - 'system time': stTimes[1], - 'children user time': stTimes[2], - 'children system time': stTimes[3], - 'elapsed real time': stTimes[4] - } - - return S_OK(dInfo) - - auth_echo = ['all'] - - @staticmethod - def export_echo(data): - """ - This method used for testing the performance of a service - """ - return S_OK(data) - - auth_whoami = ['authenticated'] - - def export_whoami(self): - """ - A simple whoami, returns all credential dictionary, except certificate chain object. - """ - credDict = self.srv_getRemoteCredentials() - if 'x509Chain' in credDict: - # Not serializable - del credDict['x509Chain'] - return S_OK(credDict) - -#### -# -# Utilities methods, some getters. -# From DIRAC.Core.DISET.requestHandler to get same interface in the handlers. -# Adapted for Tornado. -# These method are copied from DISET RequestHandler, they are not all used when i'm writing -# these lines. I rewrite them for Tornado to get them ready when a new HTTPS service need them -# -#### - - @classmethod - def srv_getCSOption(cls, optionName, defaultValue=False): - """ - Get an option from the CS section of the services - - :return: Value for serviceSection/optionName in the CS being defaultValue the default - """ - if optionName[0] == "/": - return gConfig.getValue(optionName, defaultValue) - for csPath in cls._serviceInfoDict['csPaths']: - result = gConfig.getOption("%s/%s" % (csPath, optionName, ), defaultValue) - if result['OK']: - return result['Value'] - return defaultValue - - def getCSOption(self, optionName, defaultValue=False): - """ - Just for keeping same public interface - """ - return self.srv_getCSOption(optionName, defaultValue) - - def srv_getRemoteAddress(self): - """ - Get the address of the remote peer. - - :return: Address of remote peer. - """ - - remote_ip = self.request.remote_ip - # Although it would be trivial to add this attribute in _HTTPRequestContext, - # Tornado won't release anymore 5.1 series, so go the hacky way - try: - remote_port = self.request.connection.stream.socket.getpeername()[1] - except Exception: # pylint: disable=broad-except - remote_port = 0 - - return (remote_ip, remote_port) - - def getRemoteAddress(self): - """ - Just for keeping same public interface - """ - return self.srv_getRemoteAddress() - - def srv_getRemoteCredentials(self): - """ - Get the credentials of the remote peer. - - :return: Credentials dictionary of remote peer. - """ - return self.credDict - - def getRemoteCredentials(self): - """ - Get the credentials of the remote peer. - - :return: Credentials dictionary of remote peer. - """ - return self.credDict - - def srv_getFormattedRemoteCredentials(self): - """ - Return the DN of user - - Mostly copy paste from - :py:meth:`DIRAC.Core.DISET.private.Transports.BaseTransport.BaseTransport.getFormattedCredentials` - - Note that the information will be complete only once the AuthManager was called - """ - address = self.getRemoteAddress() - peerId = "" - # Depending on where this is call, it may be that credDict is not yet filled. - # (reminder: AuthQuery fills part of it..) - try: - peerId = "[%s:%s]" % (self.credDict['group'], self.credDict['username']) - except AttributeError: - pass - - if address[0].find(":") > -1: - return "([%s]:%s)%s" % (address[0], address[1], peerId) - return "(%s:%s)%s" % (address[0], address[1], peerId) - -# def getFormattedCredentials(self): -# peerCreds = self.getConnectingCredentials() -# address = self.getRemoteAddress() -# if 'username' in peerCreds: -# peerId = "[%s:%s]" % (peerCreds['group'], peerCreds['username']) -# else: -# peerId = "" -# if address[0].find(":") > -1: -# return "([%s]:%s)%s" % (address[0], address[1], peerId) -# return "(%s:%s)%s" % (address[0], address[1], peerId) - - def srv_getServiceName(self): - """ - Return the service name - """ - return self._serviceInfoDict['serviceName'] - - def srv_getURL(self): - """ - Return the URL - """ - return self.request.path + return decode(args_encoded)[0] diff --git a/src/DIRAC/Core/Tornado/Web/Conf.py b/src/DIRAC/Core/Tornado/Web/Conf.py new file mode 100644 index 00000000000..660d9a7d2f8 --- /dev/null +++ b/src/DIRAC/Core/Tornado/Web/Conf.py @@ -0,0 +1,426 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import uuid +import tempfile +import tornado.process + +from diraccfg import CFG + +from DIRAC import gConfig, rootPath, gLogger +from DIRAC.Core.Security import Locations, X509Chain, X509CRL +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals + +__RCSID__ = "$Id$" + +BASECS = "/WebApp" + + +def getCSValue(opt, defValue=None): + """ Get option from CS + + :param str opt: option + :param defValue: default value + + :return: defValue type result + """ + return gConfig.getValue("%s/%s" % (BASECS, opt), defValue) + + +def getCSSections(opt): + """ Get sections from CS + + :param str opt: option + + :return: S_OK(list)/S_ERROR() + """ + return gConfig.getSections("%s/%s" % (BASECS, opt)) + + +def getCSOptions(opt): + """ Get options from CS + + :param str opt: option + + :return: S_OK(list)/S_ERROR() + """ + return gConfig.getOptions("%s/%s" % (BASECS, opt)) + + +def getCSOptionsDict(opt): + """ Get options dictionary from CS + + :param str opt: option + + :return: S_OK(dict)/S_ERROR() + """ + return gConfig.getOptionsDict("%s/%s" % (BASECS, opt)) + + +def getTitle(): + """ Get title + + :return: str + """ + defVal = gConfig.getValue("/DIRAC/Configuration/Name", gConfig.getValue("/DIRAC/Setup")) + return "%s - DIRAC" % gConfig.getValue("%s/Title" % BASECS, defVal) + + +def devMode(): + """ Get development mode status + + :result: bool + """ + return getCSValue("DevelopMode", False) + + +def rootURL(): + """ Get root URL + + :return: str + """ + return getCSValue("RootURL", "/DIRAC") + + +def balancer(): + """ Get balancer + + :return: str + """ + b = getCSValue("Balancer", "").lower() + if b in ("", "none"): + return "" + return b + + +def numProcesses(): + """ Get number of processes + + :return: int + """ + return getCSValue("NumProcesses", 1) + + +def HTTPS(): + """ Get flag of enable HTTPS + + :return: bool + """ + if balancer(): + return False + return getCSValue("HTTPS/Enabled", True) + + +def HTTPPort(): + """ Get HTTP port + + :return: int + """ + if balancer(): + default = 8000 + else: + default = 8080 + procAdd = tornado.process.task_id() or 0 + return getCSValue("HTTP/Port", default) + procAdd + + +def HTTPSPort(): + """ Get HTTPS port + + :return: int + """ + return getCSValue("HTTPS/Port", 8443) + + +def HTTPSCert(): + """ Get certificate path for HTTPS + + :return: str + """ + cert = Locations.getHostCertificateAndKeyLocation() + if cert: + cert = cert[0] + else: + cert = "/opt/dirac/etc/grid-security/hostcert.pem" + return getCSValue("HTTPS/Cert", cert) + + +def HTTPSKey(): + """ Get key path for HTTPS + + :return: str + """ + key = Locations.getHostCertificateAndKeyLocation() + if key: + key = key[1] + else: + key = "/opt/dirac/etc/grid-security/hostkey.pem" + return getCSValue("HTTPS/Key", key) + + +def setup(): + """ Get setup path + + :return: str + """ + return gConfig.getValue("/DIRAC/Setup") + + +def cookieSecret(): + """ Get cookie secret + + :return: str + """ + # TODO: Store the secret somewhere + return gConfig.getValue("CookieSecret", uuid.getnode()) + + +def generateCAFile(): + """ Generate a single CA file with all the PEMs + + :return: str or bool + """ + caDir = Locations.getCAsLocation() + for fn in (os.path.join(os.path.dirname(caDir), "cas.pem"), + os.path.join(os.path.dirname(HTTPSCert()), "cas.pem"), + False): + if not fn: + fn = tempfile.mkstemp(prefix="cas.", suffix=".pem")[1] + try: + fd = open(fn, "w") + except IOError: + continue + for caFile in os.listdir(caDir): + caFile = os.path.join(caDir, caFile) + result = X509Chain.X509Chain.instanceFromFile(caFile) + if not result['OK']: + continue + chain = result['Value'] + expired = chain.hasExpired() + if not expired['OK'] or expired['Value']: + continue + fd.write(chain.dumpAllToString()['Value']) + fd.close() + return fn + return False + + +def generateRevokedCertsFile(): + """ Generate a single CA file with all the PEMs + + :return: str or bool + """ + caDir = Locations.getCAsLocation() + for fn in (os.path.join(os.path.dirname(caDir), "allRevokedCerts.pem"), + os.path.join(os.path.dirname(HTTPSCert()), "allRevokedCerts.pem"), + False): + if not fn: + fn = tempfile.mkstemp(prefix="allRevokedCerts", suffix=".pem")[1] + try: + fd = open(fn, "w") + except IOError: + continue + for caFile in os.listdir(caDir): + caFile = os.path.join(caDir, caFile) + result = X509CRL.X509CRL.instanceFromFile(caFile) + if not result['OK']: + continue + chain = result['Value'] + fd.write(chain.dumpAllToString()['Value']) + fd.close() + return fn + return False + + +def getAuthSectionForHandler(route): + """ Get auth section for handler + + :return: str + """ + return "%s/Access/%s" % (BASECS, route) + + +def getTheme(): + """ Get theme + + :return: str + """ + return getCSValue("Theme", "tabs") + + +def getIcon(): + """ Get icon path + + :return: str + """ + return getCSValue("Icon", "/static/core/img/icons/system/favicon.ico") + + +def SSLProrocol(): + """ Get ssl protocol + + :return: str + """ + return getCSValue("SSLProtcol", "") + + +def getDefaultStaticDirs(): + """ Get default static directories + + :return: list + """ + defDirs = getCSValue("DefaultStaticDirs", ['defaults', 'demo']) + if defDirs == ['None']: + return [] + return defDirs + + +def getStaticDirs(): + """ Get static directories + + :return: str + """ + return list(set(getCSValue("StaticDirs", []) + getDefaultStaticDirs())) + + +def getLogo(): + """ Get logo path + + :return: str + """ + return getCSValue("Logo", "/static/core/img/icons/system/_logo_waiting.gif") + + +def getBackgroud(): + """ Get background path + + :return: str + """ + return getCSValue("BackgroundImage", "/static/core/img/wallpapers/dirac_background_6.png") + + +def getWelcome(): + """ Get welcome + + :return: str + """ + return getCSValue("WelcomeHTML", "") + + +def bugReportURL(): + """ Get bug report URL + + :return: str + """ + return getCSValue("bugReportURL", "") + + +def getAuthNames(): + """ Get enabled id providers + + :return: S_OK(list)/S_ERROR() + """ + return getCSSections("TypeAuths") + + +def getAppSettings(app): + """ Get applications options + + :param str app: application name + + :return: S_OK(dict)/S_ERROR + """ + return gConfig.getOptionsDictRecursively("%s/%s" % (BASECS, app)) + +def loadWebCFG(): + """ Load required CFG files + """ + from pprint import pprint + pprint(gConfig.getOptionsDictRecursively('/WebApp')) + if not _loadDefaultWebCFG(): + # if we have a web.cfg under etc directory we use it, otherwise + # we use the configuration file defined by the developer + _loadWebAppCFGFiles() + print('CONF UPLOADED') + + pprint(gConfig.getOptionsDictRecursively('/WebApp')) + +def _loadWebAppCFGFiles(): + """ + Load WebApp/web.cfg definitions + """ + exts = [] + for ext in CSGlobals.getCSExtensions(): + if ext == "DIRAC": + continue + if ext[-5:] != "DIRAC": + ext = "%sDIRAC" % ext + if ext != "WebAppDIRAC": + exts.append(ext) + exts.append("DIRAC") + exts.append("WebAppDIRAC") + webCFG = CFG() + for modName in reversed(exts): + try: + modPath = imp.find_module(modName)[1] + except ImportError: + continue + gLogger.verbose("Found module %s at %s" % (modName, modPath)) + cfgPath = os.path.join(modPath, "WebApp", "web.cfg") + if not os.path.isfile(cfgPath): + gLogger.verbose("Inexistant %s" % cfgPath) + continue + try: + modCFG = CFG().loadFromFile(cfgPath) + except Exception as excp: + gLogger.error("Could not load %s: %s" % (cfgPath, excp)) + continue + gLogger.verbose("Loaded %s" % cfgPath) + expl = [Conf.BASECS] + while len(expl): + current = expl.pop(0) + if not modCFG.isSection(current): + continue + if modCFG.getOption("%s/AbsoluteDefinition" % current, False): + gLogger.verbose("%s:%s is an absolute definition" % (modName, current)) + try: + webCFG.deleteKey(current) + except BaseException: + pass + modCFG.deleteKey("%s/AbsoluteDefinition" % current) + else: + for sec in modCFG[current].listSections(): + expl.append("%s/%s" % (current, sec)) + # Add the modCFG + webCFG = webCFG.mergeWith(modCFG) + gConfig.loadCFG(webCFG) + +def _loadDefaultWebCFG(): + """ This method reloads the web.cfg file from etc/web.cfg + + :return: bool + """ + modCFG = None + cfgPath = os.path.join(rootPath, 'etc', 'web.cfg') + isLoaded = True + if not os.path.isfile(cfgPath): + isLoaded = False + else: + try: + modCFG = CFG().loadFromFile(cfgPath) + except Exception as excp: + isLoaded = False + gLogger.error("Could not load %s: %s" % (cfgPath, excp)) + + if modCFG: + if modCFG.isSection("/Website"): + gLogger.warn("%s configuration file is not correct. It is used by the old portal!" % (cfgPath)) + isLoaded = False + else: + gConfig.loadCFG(modCFG) + else: + isLoaded = False + + return isLoaded diff --git a/src/DIRAC/Core/Tornado/Web/CoreHandler.py b/src/DIRAC/Core/Tornado/Web/CoreHandler.py new file mode 100644 index 00000000000..f012878bcff --- /dev/null +++ b/src/DIRAC/Core/Tornado/Web/CoreHandler.py @@ -0,0 +1,35 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import urlparse +import tornado.web + +from DIRAC.Core.Tornado.Web import Conf + +__RCSID__ = "$Id$" + + +class CoreHandler(tornado.web.RequestHandler): + + def initialize(self, action): + self.__action = action + + def get(self, setup, group, route): + if self.__action == "addSlash": + o = urlparse.urlparse(self.request.uri) + proto = self.request.protocol + if 'X-Scheme' in self.request.headers: + proto = self.request.headers['X-Scheme'] + nurl = "%s://%s%s/" % (proto, self.request.host, o.path) + if o.query: + nurl = "%s?%s" % (nurl, o.query) + self.redirect(nurl, permanent=True) + elif self.__action == "sendToRoot": + dest = "/" + rootURL = Conf.rootURL() + if rootURL: + dest += "%s/" % rootURL.strip("/") + if setup and group: + dest += "s:%s/g:%s/" % (setup, group) + self.redirect(dest) diff --git a/src/DIRAC/Core/Tornado/Web/HandlerMgr.py b/src/DIRAC/Core/Tornado/Web/HandlerMgr.py new file mode 100644 index 00000000000..c6d1bfb35f2 --- /dev/null +++ b/src/DIRAC/Core/Tornado/Web/HandlerMgr.py @@ -0,0 +1,189 @@ +""" Basic modules for loading handlers +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import re +import imp +import inspect +import collections + +from DIRAC import S_OK, S_ERROR, rootPath, gLogger +from DIRAC.Core.Tornado.Web import Conf +from DIRAC.Core.Tornado.Web.WebHandler import WebHandler, WebSocketHandler +from DIRAC.Core.Tornado.Web.CoreHandler import CoreHandler +from DIRAC.Core.Tornado.Web.StaticHandler import StaticHandler +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader +from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals +# from DIRAC.FrameworkSystem.private.authorization import AuthServer + +__RCSID__ = "$Id$" + + +class HandlerMgr(object): + __metaclass__ = DIRACSingleton + + def __init__(self, sysService=[], baseURL="/"): + """ Constructor + + :param list sysService: DIRAC system services + :param basestring baseURL: base URL + """ + self.__baseURL = baseURL.strip("/") + if sysService and not isinstance(sysService, list): + sysService = sysService.replace(' ', '').split(',') + self.__sysServices = sysService + self.__routes = [] + self.__handlers = [] + self.__setupGroupRE = r"(?:/s:([\w-]*)/g:([\w.-]*))?" + self.__shySetupGroupRE = r"(?:/s:(?:[\w-]*)/g:(?:[\w.-]*))?" + self.log = gLogger.getSubLogger("Routing") + self.__isAuthServer = False + self.__isPortal = True + + def getPaths(self, dirName): + """ Get lists of paths for all installed and enabled extensions + + :param basestring dirName: path to handlers + + :return: list + """ + pathList = [] + extNames = CSGlobals.getCSExtensions() + if 'WebAppDIRAC' in extNames: + extNames.append(extNames.pop(extNames.index('WebAppDIRAC'))) + for extName in extNames: + if not extName.endswith('DIRAC'): + extName = "%sDIRAC" % extName + try: + modFile, modPath, desc = imp.find_module(extName) + # to match in the real root path to enabling module web extensions (static, templates...) + realModPath = os.path.realpath(modPath) + except ImportError: + continue + staticPath = os.path.join(realModPath, "WebApp", dirName) + if os.path.isdir(staticPath): + pathList.append(staticPath) + return pathList + + def __calculateRoutes(self): + """ Load all handlers and generate the routes + + :return: S_OK()/S_ERROR() + """ + ol = ObjectLoader() + hendlerList = [] + self.log.debug("Added services: %s" % ','.join(self.__sysServices)) + for origin in self.__sysServices: + result = ol.getObjects(origin, parentClass=WebHandler, recurse=True) + if not result['OK']: + return result + hendlerList += list(result['Value'].items()) + self.__handlers = collections.OrderedDict(hendlerList) + + # ['/opt/dirac/pro/WebAppExt/WebApp/static', ...] + staticPaths = self.getPaths("static") + self.log.verbose("Static paths found:\n - %s" % "\n - ".join(staticPaths)) + self.__routes = [] + + # Add some standard paths for static files + statDirectories = Conf.getStaticDirs() + self.log.info("The following static directories are used:%s" % str(statDirectories)) + for stdir in statDirectories: + pattern = '/%s/(.*)' % stdir + self.__routes.append((pattern, StaticHandler, dict(pathList=['%s/webRoot/www/%s' % (rootPath, stdir)]))) + self.log.debug(" - Static route: %s" % pattern) + + for pattern in ((r"/static/(.*)", r"/(favicon\.ico)", r"/(robots\.txt)")): + pattern = r"%s%s" % (self.__shySetupGroupRE, pattern) + if self.__baseURL: + pattern = "/%s%s" % (self.__baseURL, pattern) + self.__routes.append((pattern, StaticHandler, dict(pathList=staticPaths))) + self.log.debug(" - Static route: %s" % pattern) + for hn in self.__handlers: + self.log.info("Found handler %s" % hn) + handler = self.__handlers[hn] + if handler.__name__ == 'AuthHandler': + self.__isAuthServer = True + # CHeck it has AUTH_PROPS + if isinstance(handler.AUTH_PROPS, type(None)): + return S_ERROR("Handler %s does not have AUTH_PROPS defined. Fix it!" % hn) + # Get the root for the handler + if handler.LOCATION: + handlerRoute = handler.LOCATION.strip("/") and "/%s" % handler.LOCATION.strip("/") or '' + else: + handlerRoute = hn[len(re.sub(r".[A-z]+$", "", hn)):].replace(".", "/").replace("Handler", "") + # Add the setup group RE before + baseRoute = self.__setupGroupRE + # IF theres a base url like /DIRAC add it + baseRoute = "/%s%s" % (self.__baseURL or '', baseRoute) + # Set properly the LOCATION after calculating where it is with helpers to add group and setup later + handler.LOCATION = handlerRoute + handler.PATH_RE = re.compile("%s(%s/[A-z]+|.)" % (baseRoute, handlerRoute)) + # if handler.OVERPATH: + # handler.PATH_RE = re.compile(handler.PATH_RE.pattern + '(/[A-z0-9=-_/|]+)?') + handler.URLSCHEMA = "/%s%%(setup)s%%(group)s%%(location)s/%%(action)s" % (self.__baseURL) + if issubclass(handler, WebSocketHandler): + handler.PATH_RE = re.compile("%s(%s)" % (baseRoute, handlerRoute)) + route = "%s(%s)" % (baseRoute, handlerRoute) + self.__routes.append((route, handler)) + self.log.verbose(" - WebSocket %s -> %s" % (handlerRoute, hn)) + self.log.debug(" * %s" % route) + continue + # Look for methods that are exported + for mName, mObj in inspect.getmembers(handler): + if inspect.ismethod(mObj) and mName.find(handler.METHOD_PREFIX) == 0: + methodName = mName[len(handler.METHOD_PREFIX):] + args = getattr(handler, 'path_%s' % methodName, []) + if mName == "web_index" and handler.__name__ == 'RootHandler': + # Index methods have the bare url + self.log.verbose(" - Route %s -> %s.web_index" % (handlerRoute, hn)) + route = "%s(%s/)" % (baseRoute, handlerRoute) + self.__routes.append((route, handler)) + self.__routes.append(("%s(%s)" % (baseRoute, handlerRoute), CoreHandler, dict(action='addSlash'))) + else: + # Normal methods get the method appended without web_ + self.log.verbose(" - Route %s/%s -> %s.%s" % (handlerRoute, mName[4:], hn, mName)) + route = "%s(%s%s)" % (baseRoute, handlerRoute, '' if methodName == 'index' else ('/%s' % methodName)) + # Use request path as options/values, for ex. ../method/