From ee58655ce697b18ca7fcb6d24a64d3b860697b92 Mon Sep 17 00:00:00 2001 From: ralf Date: Mon, 27 Jun 2022 21:08:34 +0200 Subject: [PATCH] WIP ADS/LDAP account-sync: - using (simple) paged result for initial sync - incremental sync uses modification time - currently only user and no periodic sync yet --- api/src/Accounts/Sql.php | 7 +- api/src/Contacts/Ldap.php | 80 +++++++--- setup/account_import.php | 231 +++++++++++++++++++++++++++++ setup/templates/default/config.tpl | 54 +++++++ 4 files changed, 347 insertions(+), 25 deletions(-) create mode 100644 setup/account_import.php diff --git a/api/src/Accounts/Sql.php b/api/src/Accounts/Sql.php index a9a1783f4c..9a8f961dd2 100644 --- a/api/src/Accounts/Sql.php +++ b/api/src/Accounts/Sql.php @@ -188,9 +188,10 @@ class Sql * If no account_id is set in data the account is added and the new id is set in $data. * * @param array $data array with account-data - * @return int/boolean the account_id or false on error + * @param bool $force_create true: do NOT check with frontend, if account exists + * @return int|false the account_id or false on error */ - function save(&$data) + function save(&$data, $force_create=false) { $to_write = $data; unset($to_write['account_passwd']); @@ -207,7 +208,7 @@ class Sql $to_write['account_lastpwd_change'] = time(); } if ($data['mustchangepassword'] == 1) $to_write['account_lastpwd_change']=0; - if (!(int)$data['account_id'] || !$this->id2name($data['account_id'])) + if ($force_create || !(int)$data['account_id'] || !$this->id2name($data['account_id'])) { if ($to_write['account_id'] < 0) $to_write['account_id'] *= -1; diff --git a/api/src/Contacts/Ldap.php b/api/src/Contacts/Ldap.php index 4cf04f981b..a24b57f84e 100644 --- a/api/src/Contacts/Ldap.php +++ b/api/src/Contacts/Ldap.php @@ -1196,8 +1196,6 @@ class Ldap */ function _searchLDAP($_ldapContext, $_filter, $_attributes, $_addressbooktype, array $_skipPlugins=null, $order_by=null, &$start=null) { - $this->total = 0; - $_attributes[] = 'entryUUID'; $_attributes[] = 'objectClass'; $_attributes[] = 'createTimestamp'; @@ -1214,24 +1212,43 @@ class Ldap { [$offset, $num_rows] = $start; - $control = [ - [ - 'oid' => LDAP_CONTROL_SORTREQUEST, - //'iscritical' => TRUE, - 'value' => $sort_values, - ], - [ - 'oid' => LDAP_CONTROL_VLVREQUEST, - //'iscritical' => TRUE, - 'value' => [ - 'before' => 0, // Return 0 entry before target - 'after' => $num_rows-1, // total-1 - 'offset' => $offset+1, // first = 1, NOT 0! - 'count' => 0, // We have no idea how many entries there are - ] + $control[] = [ + 'oid' => LDAP_CONTROL_SORTREQUEST, + //'iscritical' => TRUE, + 'value' => $sort_values, + ]; + $control[] = [ + 'oid' => LDAP_CONTROL_VLVREQUEST, + //'iscritical' => TRUE, + 'value' => [ + 'before' => 0, // Return 0 entry before target + 'after' => $num_rows-1, // total-1 + 'offset' => $offset+1, // first = 1, NOT 0! + 'count' => 0, // We have no idea how many entries there are ] ]; } + elseif (PHP_VERSION >= 7.3 && empty($order_by) && + ($start === false || is_array($start) && count($start) === 3) && + $this->ldapServerInfo->supportedControl(LDAP_CONTROL_PAGEDRESULTS)) + { + if ($start === false) + { + $start = [false, 500, '']; + } + $control[] = [ + 'oid' => LDAP_CONTROL_PAGEDRESULTS, + //'iscritical' => TRUE, + 'value' => [ + 'size' => $start[1], + 'cookie' => $start[2], + ], + ]; + } + if (!is_array($start) || count($start) < 3 || $start[2] === '') + { + $this->total = 0; + } if($_addressbooktype == self::ALL || $_ldapContext == $this->allContactsDN) { @@ -1242,15 +1259,21 @@ class Ldap $result = @ldap_list($this->ds, $_ldapContext, $_filter, $_attributes, 0, $this->ldapLimit, null, null, $control); } if(!$result || !$entries = ldap_get_entries($this->ds, $result)) return array(); - $this->total = $entries['count']; + $this->total += $entries['count']; //error_log(__METHOD__."('$_ldapContext', '$_filter', ".array2string($_attributes).", $_addressbooktype) result of $entries[count]"); // check if given controls succeeded - if ($control && ldap_parse_result($this->ds, $result, $errcode, $matcheddn, $errmsg, $referrals, $serverctrls) && - (isset($serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count']))) + if ($control && ldap_parse_result($this->ds, $result, $errcode, $matcheddn, $errmsg, $referrals, $serverctrls)) { - $this->total = $serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count']; - $start = null; // so caller does NOT run it's own limit + if (isset($serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count'])) + { + $this->total = $serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count']; + $start = null; // so caller does NOT run it's own limit + } + elseif (isset($serverctrls[LDAP_CONTROL_PAGEDRESULTS]['value']['cookie'])) + { + $start[2] = $serverctrls[LDAP_CONTROL_PAGEDRESULTS]['value']['cookie']; + } } foreach($entries as $i => $entry) @@ -1315,6 +1338,19 @@ class Ldap } $contacts[] = $contact; } + + // if we have a non-empty cookie from paged results, continue reading from the server + while (is_array($start) && count($start) === 3 && $start[0] === false && $start[2] !== '') + { + foreach($this->_searchLDAP($_ldapContext, $_filter, $_attributes, $_addressbooktype, $_skipPlugins, $order_by, $start) as $contact) + { + $contacts[] = $contact; + } + } + if (is_array($start) && $start[0] === false) + { + $start = false; + } return $contacts; } diff --git a/setup/account_import.php b/setup/account_import.php new file mode 100644 index 0000000000..483402b49f --- /dev/null +++ b/setup/account_import.php @@ -0,0 +1,231 @@ + + * @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License + */ + +use EGroupware\Api; + +include('./inc/functions.inc.php'); + +// Authorize the user to use setup app and load the database +if (!$GLOBALS['egw_setup']->auth('Config') || $_POST['cancel']) +{ + Header('Location: index.php'); + exit; +} +// Does not return unless user is authorized + +// check CSRF token for POST requests with any content (setup uses empty POST to call its modules!) +if ($_SERVER['REQUEST_METHOD'] == 'POST' && $_POST) +{ + Api\Csrf::validate($_POST['csrf_token'], __FILE__); +} + +// determine from where we migrate to what +if (!is_object($GLOBALS['egw_setup']->db)) +{ + $GLOBALS['egw_setup']->loaddb(); +} +// Load configuration values account_repository and auth_type, as setup has not yet done so +foreach($GLOBALS['egw_setup']->db->select($GLOBALS['egw_setup']->config_table,'config_name,config_value', + "config_name LIKE 'ldap%' OR config_name LIKE 'account_%' OR config_name LIKE '%encryption%' OR ". + "config_name IN ('auth_type','install_id','mail_suffix') OR config_name LIKE 'ads_%' OR config_name LIKE 'account_import_%'", + __LINE__,__FILE__) as $row) +{ + $GLOBALS['egw_info']['server'][$row['config_name']] = $row['config_value']; +} + +try { + if (!in_array($source = $GLOBALS['egw_info']['server']['account_import_source'], ['ldap', 'ads'])) + { + throw new \InvalidArgumentException("Invalid account_import_source='{$GLOBALS['egw_info']['server']['account_import_source']}'!"); + } + if (!in_array($type = $GLOBALS['egw_info']['server']['account_import_type'], ['users', 'users_groups'])) + { + throw new \InvalidArgumentException("Invalid account_import_type='{$GLOBALS['egw_info']['server']['account_import_type']}'!"); + } + if (!in_array($delete = $GLOBALS['egw_info']['server']['account_import_delete'], ['yes', 'deactivate', 'no'])) + { + throw new \InvalidArgumentException("Invalid account_import_delete='{$GLOBALS['egw_info']['server']['account_import_delete']}'!"); + } + + if (!($initial_import=!empty($_REQUEST['initial'])) && empty($GLOBALS['egw_info']['server']['account_import_lastrun'])) + { + throw new \InvalidArgumentException(lang("You need to run the inital import first!")); + } + + $class = 'EGroupware\\Api\\Contacts\\'.ucfirst($source); + /** @var Api\Contacts\Ldap $contacts */ + $contacts = new $class($GLOBALS['egw_info']['server']); + $contacts_sql = new Api\Contacts\Sql(); + + $class = 'EGroupware\\Api\\Accounts\\'.ucfirst($source); + /** @var Api\Accounts\Ldap $accounts */ + $accounts = new $class(new Api\Accounts(['account_repository' => $source]+$GLOBALS['egw_info']['server'])); + $accounts_sql = new Api\Accounts\Sql(new Api\Accounts(['account_repository' => 'sql']+$GLOBALS['egw_info']['server'])); + Api\Accounts::cache_invalidate(); // to not get any cached data eg. from the wrong backend + + $filter = [ + 'owner' => '0', + ]; + if (!$initial_import) + { + $filter[] = 'modified>='.$GLOBALS['egw_info']['server']['account_import_lastrun']; + } + $last_modified = null; + $start_import = time(); + $created = $updated = $uptodate = $errors = 0; + $cookie = ''; + $start = ['', 5, &$cookie]; + do + { + foreach ($contacts->search('', false, '', 'account_lid', '', '', 'AND', $start, $filter) as $contact) + { + $new = null; + if (!isset($last_modified) || (int)$last_modified < (int)$contact['modified']) + { + $last_modified = $contact['modified']; + } + $account = $accounts->read($contact['account_id']); + echo "

" . json_encode($contact + $account) . "

\n"; + // check if account exists in sql + if (!($account_id = $accounts_sql->name2id($account['account_lid']))) + { + $sql_account = $account; + if ($accounts_sql->save($account, true) > 0) + { + echo "

Successful created user '$account[account_lid]' (#$account_id)
\n"; + $new; + } + else + { + echo "

Error creaing user '$account[account_lid]' (#$account_id)
\n"; + $errors++; + continue; + } + } + elseif ($account_id < 0) + { + throw new \Exception("User '$account[account_lid]' already exists as group!"); + } + elseif (!($sql_account = $accounts_sql->read($account_id))) + { + throw new \Exception("User '$account[account_lid]' (#$account_id) should exist, but not found!"); + } + else + { + // ignore LDAP specific fields, and empty fields + $relevant = array_filter(array_intersect_key($account, $sql_account), static function ($attr) { + return $attr !== null && $attr !== ''; + }); + unset($relevant['person_id']); // is always different as it's the UID, no need to consider + $to_update = $relevant + $sql_account; + // fix accounts without firstname + if (!isset($to_update['account_firstname']) && $to_update['account_lastname'] === $to_update['account_fullname']) + { + $to_update['account_firstname'] = null; + } + if (($diff = array_diff_assoc($to_update, $sql_account))) + { + if ($accounts_sql->save($to_update) > 0) + { + echo "

Successful updated user '$account[account_lid]' (#$account_id): " . json_encode($diff, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . "
\n"; + if (!$new) $new = false; + } + else + { + echo "

Error updating user '$account[account_lid]' (#$account_id)
\n"; + $errors++; + continue; + } + } + else + { + echo "

User '$account[account_lid]' (#$account_id) already up to date
\n"; + } + } + if (!($sql_contact = $contacts_sql->read(['account_id' => $account_id]))) + { + $sql_contact = $contact; + unset($sql_contact['id']); // LDAP contact-id is the UID! + if (!$contacts_sql->save($sql_contact)) + { + $sql_contact['id'] = $contacts_sql->data['id']; + echo "Successful created contact for user '$account[account_lid]' (#$account_id)

\n"; + $new = true; + } + else + { + echo "Error creating contact for user '$account[account_lid]' (#$account_id)

\n"; + $errors++; + continue; + } + } + else + { + $to_update = array_merge($sql_contact, array_filter($contact, static function ($attr) { + return $attr !== null && $attr !== ''; + })); + $to_update['id'] = $sql_contact['id']; + if (($diff = array_diff_assoc($to_update, $sql_contact))) + { + if ($contacts_sql->save($to_update) === 0) + { + echo "Successful updated contact data of '$account[account_lid]' (#$account_id): ".json_encode($diff, JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE)."

\n"; + if (!$new) $new = false; + } + else + { + echo "Error updating contact data of '$account[account_lid]' (#$account_id)

\n"; + ++$errors; + continue; + } + } + else + { + echo "Contact data of '$account[account_lid]' (#$account_id) already up to date

\n"; + } + } + if ($new) + { + ++$created; + } + elseif ($new === false) + { + ++$updated; + } + else + { + ++$uptodate; + } + } + } + while ($start[2] !== ''); + $last_run = max($start_import-1, $last_modified); + Api\Config::save_value('account_import_lastrun', $last_run, 'phpgwapi'); + $str = gmdate('Y-m-d H:i:s', $last_run). ' UTC'; + if ($created || $updated || $errors) + { + echo "

Created $created and updated $updated users, with $errors errors.

"; + } + else + { + echo "

All users are up-to-date.

\n"; + } + if (!$errors) + { + echo "

Setting new incremental import time to: $str ($last_run)

\n"; + } +} +catch (\Exception $e) { + echo "

".$e->getMessage()."

\n"; + http_response_code(500); + exit; +} \ No newline at end of file diff --git a/setup/templates/default/config.tpl b/setup/templates/default/config.tpl index 6c1266163b..0735b56d32 100644 --- a/setup/templates/default/config.tpl +++ b/setup/templates/default/config.tpl @@ -449,6 +449,60 @@   + + {lang_Periodic_import_from_ADS_or_LDAP_into_EGroupware_database}: + + + + {lang_Source_(must_be_configured_above)}: + + + + + + {lang_What_to_import?}: + + + + + + {lang_What_to_do_in_EGroupware_if_an_user_get_deleted?}: + + + + + + {lang_How_frequent_should_the_import_run?}: + + {lang_Every} + + {lang_hours_at} + + + + + {lang_You_must_save_AND_run_an_inital_import,_before_the_periodic_import_will_start}: + + + + {lang_We_strongly_recomment_to_run_a_DB_backup_BEFORE_running_the_import!} + + + + +   + + {lang_If_using_Mail_authentication}: