Merge pull request #27 from mwestphall/INF-1060-member-removals

INF-1060: Generate project mapfile from topology/LDAP instead of COManage

Author: brianhlin

comanage_utils.py

@@ -6,7 +6,8 @@
 import time
 import urllib.error
 import urllib.request
-from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC
+from ldap3 import Server, Connection, ALL, SAFE_SYNC, Tls
+from dataclasses import dataclass
 
 #PRODUCTION VALUES
 
@@ -16,6 +17,7 @@
 PRODUCTION_OSG_CO_ID = 7
 PRODUCTION_UNIX_CLUSTER_ID = 1
 PRODUCTION_LDAP_TARGET_ID = 6
+LDAP_BASE_DN = "o=OSG,o=CO,dc=cilogon,dc=org"
 
 #TEST VALUES
 
@@ -69,6 +71,14 @@ def mkauthstr(user, passwd):
     return encodebytes(raw_authstr.encode()).decode().replace("\n", "")
 
 
+def get_ldap_authtok(ldap_authfile):
+    if ldap_authfile is not None:
+        ldap_authtok = open(ldap_authfile).readline().strip()
+    else:
+        raise PermissionError
+    return ldap_authtok
+
+
 def mkrequest(method, target, data, endpoint, authstr, **kw):
     url = os.path.join(endpoint, target)
     if kw:
@@ -165,16 +175,43 @@ def get_datalist(data, listname):
     return data[listname] if data else []
 
 
+class LDAPSearch:
+    """ Wrapper class for LDAP searches. """
+    server: Server = None
+    connection: Connection = None
+
+    def __init__(self, ldap_server, ldap_user, ldap_authtok):
+        self.server = Server(ldap_server, get_info=ALL)
+        self.connection = Connection(self.server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
+
+    def search(self, ou, filter_str, attrs):
+        _, _, response, _ = self.connection.search(f"ou={ou},{LDAP_BASE_DN}", filter_str, attributes=attrs)
+        return response
+
 def get_ldap_groups(ldap_server, ldap_user, ldap_authtok):
     ldap_group_osggids = set()
-    server = Server(ldap_server, get_info=ALL)
-    connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
-    _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES)
+    searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok)
+    response = searcher.search("groups", "(cn=*)", ["gidNumber"])
     for group in response:
         ldap_group_osggids.add(group["attributes"]["gidNumber"])
     return ldap_group_osggids
 
 
+def get_ldap_active_users_and_groups(ldap_server, ldap_user, ldap_authtok, filter_group_name=None):
+    """ Retrieve a dictionary of active users from LDAP, with their group memberships. """
+    ldap_active_users = dict()
+    filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None 
+                  else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))")
+
+    searcher = LDAPSearch(ldap_server, ldap_user, ldap_authtok)
+    response = searcher.search("people", filter_str, ["employeeNumber", "isMemberOf"])
+
+    for person in response:
+        ldap_active_users[person["attributes"]["employeeNumber"]] = person["attributes"].get("isMemberOf", [])
+
+    return ldap_active_users
+
+
 def identifier_from_list(id_list, id_type):
     id_type_list = [id["Type"] for id in id_list]
     try:

osg-comanage-project-usermap.py

@@ -4,28 +4,37 @@
 import re
 import sys
 import getopt
-import collections
+import requests
 import comanage_utils as utils
 
 
 SCRIPT = os.path.basename(__file__)
-ENDPOINT = "https://registry.cilogon.org/registry/"
-OSG_CO_ID = 7
+ENDPOINT = "https://registry-test.cilogon.org/registry/"
+TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/"
+LDAP_SERVER = "ldaps://ldap-test.cilogon.org"
+LDAP_USER = "uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org"
+OSG_CO_ID = 8
+CACHE_FILENAME = "COmanage_Projects_cache.txt"
+CACHE_LIFETIME_HOURS = 0.5
 
 
 _usage = f"""\
-usage: [PASS=...] {SCRIPT} [OPTIONS]
+usage: {SCRIPT} [OPTIONS]
 
 OPTIONS:
   -u USER[:PASS]      specify USER and optionally PASS on command line
   -c OSG_CO_ID        specify OSG CO ID (default = {OSG_CO_ID})
+  -s LDAP_SERVER      specify LDAP server to read data from
+  -l LDAP_USER        specify LDAP user for reading data from LDAP server
+  -a ldap_authfile    specify path to file to open and read LDAP authtok
   -d passfd           specify open fd to read PASS
   -f passfile         specify path to file to open and read PASS
   -e ENDPOINT         specify REST endpoint
                         (default = {ENDPOINT})
   -o outfile          specify output file (default: write to stdout)
   -g filter_group     filter users by group name (eg, 'ap1-login')
-  -l localmaps        specify a comma-delimited list of local HTCondor mapfiles to merge into outfile
+  -m localmaps        specify a comma-delimited list of local HTCondor mapfiles to merge into outfile
+  -n min_users        Specify minimum number of users required to update the output file (default: 100)
   -h                  display this help text
 
 PASS for USER is taken from the first of:
@@ -49,7 +58,11 @@ class Options:
     osg_co_id = OSG_CO_ID
     outfile = None
     authstr = None
+    ldap_server = LDAP_SERVER
+    ldap_user = LDAP_USER
+    ldap_authtok = None
     filtergrp = None
+    min_users = 100 # Bail out before updating the file if we have fewer than this many users
     localmaps = []
 
 
@@ -62,35 +75,12 @@ def get_osg_co_groups__map():
     #print("get_osg_co_groups__map()")
     resp_data = utils.get_osg_co_groups(options.osg_co_id, options.endpoint, options.authstr)
     data = utils.get_datalist(resp_data, "CoGroups")
-    return { g["Id"]: g["Name"] for g in data }
-
-
-def co_group_is_ospool(gid):
-    #print(f"co_group_is_ospool({gid})")
-    resp_data = utils.get_co_group_identifiers(gid, options.endpoint, options.authstr)
-    data = utils.get_datalist(resp_data, "Identifiers")
-    return any( i["Type"] == "ospoolproject" for i in data )
-
-
-def get_co_group_members__pids(gid):
-    #print(f"get_co_group_members__pids({gid})")
-    resp_data = utils.get_co_group_members(gid,  options.endpoint, options.authstr)
-    data = utils.get_datalist(resp_data, "CoGroupMembers")
-    # For INF-1060: Temporary Fix until "The Great Project Provisioning" is finished
-    return [ m["Person"]["Id"] for m in data if m["Member"] == True]
-
-
-def get_co_person_osguser(pid):
-    #print(f"get_co_person_osguser({pid})")
-    resp_data = utils.get_co_person_identifiers(pid, options.endpoint, options.authstr)
-    data = utils.get_datalist(resp_data, "Identifiers")
-    typemap = { i["Type"]: i["Identifier"] for i in data }
-    return typemap.get("osguser")
+    return { g["Name"]: g["Id"] for g in data }
 
 
 def parse_options(args):
     try:
-        ops, args = getopt.getopt(args, 'u:c:d:f:g:e:o:l:h')
+        ops, args = getopt.getopt(args, 'u:c:s:l:a:d:f:g:e:o:h:n:m')
     except getopt.GetoptError:
         usage()
 
@@ -99,21 +89,27 @@ def parse_options(args):
 
     passfd = None
     passfile = None
+    ldap_authfile = None
 
     for op, arg in ops:
         if op == '-h': usage()
         if op == '-u': options.user       = arg
         if op == '-c': options.osg_co_id  = int(arg)
+        if op == '-s': options.ldap_server= arg
+        if op == '-l': options.ldap_user  = arg
+        if op == '-a': ldap_authfile      = arg
         if op == '-d': passfd             = int(arg)
         if op == '-f': passfile           = arg
         if op == '-e': options.endpoint   = arg
         if op == '-o': options.outfile    = arg
         if op == '-g': options.filtergrp  = arg
-        if op == '-l': options.localmaps = arg.split(",")
+        if op == '-m': options.localmaps  = arg.split(",")
+        if op == '-n': options.min_users  = int(arg)
 
     try:
         user, passwd = utils.getpw(options.user, passfd, passfile)
         options.authstr = utils.mkauthstr(user, passwd)
+        options.ldap_authtok = utils.get_ldap_authtok(ldap_authfile)
     except PermissionError:
         usage("PASS required")
 
@@ -123,36 +119,18 @@ def _deduplicate_list(items):
     """
     return list(dict.fromkeys(items))
 
-def gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser):
-    pid_gids = collections.defaultdict(list)
-
-    for gid in gid_pids:
-        for pid in gid_pids[gid]:
-            if pid_osguser[pid] is not None and gid not in pid_gids[pid]:
-                pid_gids[pid].append(gid)
-
-    return pid_gids
-
-
-def filter_by_group(pid_gids, groups, filter_group_name):
-    groups_idx = { v: k for k,v in groups.items() }
-    filter_gid = groups_idx[filter_group_name]  # raises KeyError if missing
-    filter_group_pids = set(get_co_group_members__pids(filter_gid))
-    return { p: g for p,g in pid_gids.items() if p in filter_group_pids }
-
-
 def get_osguser_groups(filter_group_name=None):
-    groups = get_osg_co_groups__map()
-    ospool_gids = filter(co_group_is_ospool, groups)
-    gid_pids = { gid: get_co_group_members__pids(gid) for gid in ospool_gids }
-    all_pids = set( pid for gid in gid_pids for pid in gid_pids[gid] )
-    pid_osguser = { pid: get_co_person_osguser(pid) for pid in all_pids }
-    pid_gids = gid_pids_to_osguser_pid_gids(gid_pids, pid_osguser)
-    if filter_group_name is not None:
-        pid_gids = filter_by_group(pid_gids, groups, filter_group_name)
-
-    return { pid_osguser[pid]: map(groups.get, gids)
-             for pid, gids in pid_gids.items() }
+    ldap_users = utils.get_ldap_active_users_and_groups(options.ldap_server, options.ldap_user, options.ldap_authtok, filter_group_name)
+    topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json()
+    project_names = topology_projects.keys()
+    
+    # Get COManage group IDs to preserve ordering from pre-LDAP migration script behavior
+    groups_ids = get_osg_co_groups__map()
+    return {
+        user: sorted([g for g in groups if g in project_names], key = lambda g: groups_ids.get(g, 0)) 
+        for user, groups in ldap_users.items()
+        if any(g in project_names for g in groups)
+    }
 
 
 def parse_localmap(inputfile):
@@ -204,6 +182,9 @@ def main(args):
         maps.append(parse_localmap(localmap))
     osguser_groups_merged = merge_maps(maps)
 
+    # Sanity check, confirm we have generated a "sane" amount of user -> group mappings
+    if len(osguser_groups_merged) < options.min_users:
+        raise RuntimeError(f"Refusing to update output file: only {len(osguser_groups_merged)} users found")
     print_usermap(osguser_groups_merged)