WIP ADS/LDAP account-sync:

- using (simple) paged result for initial sync
- incremental sync uses modification time
- currently only user and no periodic sync yet
This commit is contained in:
ralf 2022-06-27 21:08:34 +02:00
parent bb48e4ecee
commit ee58655ce6
4 changed files with 347 additions and 25 deletions

View File

@ -188,9 +188,10 @@ class Sql
* If no account_id is set in data the account is added and the new id is set in $data.
*
* @param array $data array with account-data
* @return int/boolean the account_id or false on error
* @param bool $force_create true: do NOT check with frontend, if account exists
* @return int|false the account_id or false on error
*/
function save(&$data)
function save(&$data, $force_create=false)
{
$to_write = $data;
unset($to_write['account_passwd']);
@ -207,7 +208,7 @@ class Sql
$to_write['account_lastpwd_change'] = time();
}
if ($data['mustchangepassword'] == 1) $to_write['account_lastpwd_change']=0;
if (!(int)$data['account_id'] || !$this->id2name($data['account_id']))
if ($force_create || !(int)$data['account_id'] || !$this->id2name($data['account_id']))
{
if ($to_write['account_id'] < 0) $to_write['account_id'] *= -1;

View File

@ -1196,8 +1196,6 @@ class Ldap
*/
function _searchLDAP($_ldapContext, $_filter, $_attributes, $_addressbooktype, array $_skipPlugins=null, $order_by=null, &$start=null)
{
$this->total = 0;
$_attributes[] = 'entryUUID';
$_attributes[] = 'objectClass';
$_attributes[] = 'createTimestamp';
@ -1214,24 +1212,43 @@ class Ldap
{
[$offset, $num_rows] = $start;
$control = [
[
'oid' => LDAP_CONTROL_SORTREQUEST,
//'iscritical' => TRUE,
'value' => $sort_values,
],
[
'oid' => LDAP_CONTROL_VLVREQUEST,
//'iscritical' => TRUE,
'value' => [
'before' => 0, // Return 0 entry before target
'after' => $num_rows-1, // total-1
'offset' => $offset+1, // first = 1, NOT 0!
'count' => 0, // We have no idea how many entries there are
]
$control[] = [
'oid' => LDAP_CONTROL_SORTREQUEST,
//'iscritical' => TRUE,
'value' => $sort_values,
];
$control[] = [
'oid' => LDAP_CONTROL_VLVREQUEST,
//'iscritical' => TRUE,
'value' => [
'before' => 0, // Return 0 entry before target
'after' => $num_rows-1, // total-1
'offset' => $offset+1, // first = 1, NOT 0!
'count' => 0, // We have no idea how many entries there are
]
];
}
elseif (PHP_VERSION >= 7.3 && empty($order_by) &&
($start === false || is_array($start) && count($start) === 3) &&
$this->ldapServerInfo->supportedControl(LDAP_CONTROL_PAGEDRESULTS))
{
if ($start === false)
{
$start = [false, 500, ''];
}
$control[] = [
'oid' => LDAP_CONTROL_PAGEDRESULTS,
//'iscritical' => TRUE,
'value' => [
'size' => $start[1],
'cookie' => $start[2],
],
];
}
if (!is_array($start) || count($start) < 3 || $start[2] === '')
{
$this->total = 0;
}
if($_addressbooktype == self::ALL || $_ldapContext == $this->allContactsDN)
{
@ -1242,15 +1259,21 @@ class Ldap
$result = @ldap_list($this->ds, $_ldapContext, $_filter, $_attributes, 0, $this->ldapLimit, null, null, $control);
}
if(!$result || !$entries = ldap_get_entries($this->ds, $result)) return array();
$this->total = $entries['count'];
$this->total += $entries['count'];
//error_log(__METHOD__."('$_ldapContext', '$_filter', ".array2string($_attributes).", $_addressbooktype) result of $entries[count]");
// check if given controls succeeded
if ($control && ldap_parse_result($this->ds, $result, $errcode, $matcheddn, $errmsg, $referrals, $serverctrls) &&
(isset($serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count'])))
if ($control && ldap_parse_result($this->ds, $result, $errcode, $matcheddn, $errmsg, $referrals, $serverctrls))
{
$this->total = $serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count'];
$start = null; // so caller does NOT run it's own limit
if (isset($serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count']))
{
$this->total = $serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count'];
$start = null; // so caller does NOT run it's own limit
}
elseif (isset($serverctrls[LDAP_CONTROL_PAGEDRESULTS]['value']['cookie']))
{
$start[2] = $serverctrls[LDAP_CONTROL_PAGEDRESULTS]['value']['cookie'];
}
}
foreach($entries as $i => $entry)
@ -1315,6 +1338,19 @@ class Ldap
}
$contacts[] = $contact;
}
// if we have a non-empty cookie from paged results, continue reading from the server
while (is_array($start) && count($start) === 3 && $start[0] === false && $start[2] !== '')
{
foreach($this->_searchLDAP($_ldapContext, $_filter, $_attributes, $_addressbooktype, $_skipPlugins, $order_by, $start) as $contact)
{
$contacts[] = $contact;
}
}
if (is_array($start) && $start[0] === false)
{
$start = false;
}
return $contacts;
}

231
setup/account_import.php Normal file
View File

@ -0,0 +1,231 @@
<?php
/**
* EGroupware Setup - Account import from LDAP (incl. ADS) to SQL
*
* The migration is done from the account-repository configured for EGroupware!
*
* @link http://www.egroupware.org
* @package setup
* @author Ralf Becker <RalfBecker-AT-outdoor-training.de>
* @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License
*/
use EGroupware\Api;
include('./inc/functions.inc.php');
// Authorize the user to use setup app and load the database
if (!$GLOBALS['egw_setup']->auth('Config') || $_POST['cancel'])
{
Header('Location: index.php');
exit;
}
// Does not return unless user is authorized
// check CSRF token for POST requests with any content (setup uses empty POST to call its modules!)
if ($_SERVER['REQUEST_METHOD'] == 'POST' && $_POST)
{
Api\Csrf::validate($_POST['csrf_token'], __FILE__);
}
// determine from where we migrate to what
if (!is_object($GLOBALS['egw_setup']->db))
{
$GLOBALS['egw_setup']->loaddb();
}
// Load configuration values account_repository and auth_type, as setup has not yet done so
foreach($GLOBALS['egw_setup']->db->select($GLOBALS['egw_setup']->config_table,'config_name,config_value',
"config_name LIKE 'ldap%' OR config_name LIKE 'account_%' OR config_name LIKE '%encryption%' OR ".
"config_name IN ('auth_type','install_id','mail_suffix') OR config_name LIKE 'ads_%' OR config_name LIKE 'account_import_%'",
__LINE__,__FILE__) as $row)
{
$GLOBALS['egw_info']['server'][$row['config_name']] = $row['config_value'];
}
try {
if (!in_array($source = $GLOBALS['egw_info']['server']['account_import_source'], ['ldap', 'ads']))
{
throw new \InvalidArgumentException("Invalid account_import_source='{$GLOBALS['egw_info']['server']['account_import_source']}'!");
}
if (!in_array($type = $GLOBALS['egw_info']['server']['account_import_type'], ['users', 'users_groups']))
{
throw new \InvalidArgumentException("Invalid account_import_type='{$GLOBALS['egw_info']['server']['account_import_type']}'!");
}
if (!in_array($delete = $GLOBALS['egw_info']['server']['account_import_delete'], ['yes', 'deactivate', 'no']))
{
throw new \InvalidArgumentException("Invalid account_import_delete='{$GLOBALS['egw_info']['server']['account_import_delete']}'!");
}
if (!($initial_import=!empty($_REQUEST['initial'])) && empty($GLOBALS['egw_info']['server']['account_import_lastrun']))
{
throw new \InvalidArgumentException(lang("You need to run the inital import first!"));
}
$class = 'EGroupware\\Api\\Contacts\\'.ucfirst($source);
/** @var Api\Contacts\Ldap $contacts */
$contacts = new $class($GLOBALS['egw_info']['server']);
$contacts_sql = new Api\Contacts\Sql();
$class = 'EGroupware\\Api\\Accounts\\'.ucfirst($source);
/** @var Api\Accounts\Ldap $accounts */
$accounts = new $class(new Api\Accounts(['account_repository' => $source]+$GLOBALS['egw_info']['server']));
$accounts_sql = new Api\Accounts\Sql(new Api\Accounts(['account_repository' => 'sql']+$GLOBALS['egw_info']['server']));
Api\Accounts::cache_invalidate(); // to not get any cached data eg. from the wrong backend
$filter = [
'owner' => '0',
];
if (!$initial_import)
{
$filter[] = 'modified>='.$GLOBALS['egw_info']['server']['account_import_lastrun'];
}
$last_modified = null;
$start_import = time();
$created = $updated = $uptodate = $errors = 0;
$cookie = '';
$start = ['', 5, &$cookie];
do
{
foreach ($contacts->search('', false, '', 'account_lid', '', '', 'AND', $start, $filter) as $contact)
{
$new = null;
if (!isset($last_modified) || (int)$last_modified < (int)$contact['modified'])
{
$last_modified = $contact['modified'];
}
$account = $accounts->read($contact['account_id']);
echo "<p>" . json_encode($contact + $account) . "</p>\n";
// check if account exists in sql
if (!($account_id = $accounts_sql->name2id($account['account_lid'])))
{
$sql_account = $account;
if ($accounts_sql->save($account, true) > 0)
{
echo "<p>Successful created user '$account[account_lid]' (#$account_id)<br/>\n";
$new;
}
else
{
echo "<p><b>Error creaing user '$account[account_lid]' (#$account_id)<br/>\n";
$errors++;
continue;
}
}
elseif ($account_id < 0)
{
throw new \Exception("User '$account[account_lid]' already exists as group!");
}
elseif (!($sql_account = $accounts_sql->read($account_id)))
{
throw new \Exception("User '$account[account_lid]' (#$account_id) should exist, but not found!");
}
else
{
// ignore LDAP specific fields, and empty fields
$relevant = array_filter(array_intersect_key($account, $sql_account), static function ($attr) {
return $attr !== null && $attr !== '';
});
unset($relevant['person_id']); // is always different as it's the UID, no need to consider
$to_update = $relevant + $sql_account;
// fix accounts without firstname
if (!isset($to_update['account_firstname']) && $to_update['account_lastname'] === $to_update['account_fullname'])
{
$to_update['account_firstname'] = null;
}
if (($diff = array_diff_assoc($to_update, $sql_account)))
{
if ($accounts_sql->save($to_update) > 0)
{
echo "<p>Successful updated user '$account[account_lid]' (#$account_id): " . json_encode($diff, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . "<br/>\n";
if (!$new) $new = false;
}
else
{
echo "<p><b>Error updating user '$account[account_lid]' (#$account_id)</b><br/>\n";
$errors++;
continue;
}
}
else
{
echo "<p>User '$account[account_lid]' (#$account_id) already up to date<br/>\n";
}
}
if (!($sql_contact = $contacts_sql->read(['account_id' => $account_id])))
{
$sql_contact = $contact;
unset($sql_contact['id']); // LDAP contact-id is the UID!
if (!$contacts_sql->save($sql_contact))
{
$sql_contact['id'] = $contacts_sql->data['id'];
echo "Successful created contact for user '$account[account_lid]' (#$account_id)</p>\n";
$new = true;
}
else
{
echo "<b>Error creating contact for user '$account[account_lid]' (#$account_id)</b></p>\n";
$errors++;
continue;
}
}
else
{
$to_update = array_merge($sql_contact, array_filter($contact, static function ($attr) {
return $attr !== null && $attr !== '';
}));
$to_update['id'] = $sql_contact['id'];
if (($diff = array_diff_assoc($to_update, $sql_contact)))
{
if ($contacts_sql->save($to_update) === 0)
{
echo "Successful updated contact data of '$account[account_lid]' (#$account_id): ".json_encode($diff, JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE)."</p>\n";
if (!$new) $new = false;
}
else
{
echo "<b>Error updating contact data of '$account[account_lid]' (#$account_id)</b></p>\n";
++$errors;
continue;
}
}
else
{
echo "Contact data of '$account[account_lid]' (#$account_id) already up to date</p>\n";
}
}
if ($new)
{
++$created;
}
elseif ($new === false)
{
++$updated;
}
else
{
++$uptodate;
}
}
}
while ($start[2] !== '');
$last_run = max($start_import-1, $last_modified);
Api\Config::save_value('account_import_lastrun', $last_run, 'phpgwapi');
$str = gmdate('Y-m-d H:i:s', $last_run). ' UTC';
if ($created || $updated || $errors)
{
echo "<p><b>Created $created and updated $updated users, with $errors errors.</b></p>";
}
else
{
echo "<p><b>All users are up-to-date.</b></p>\n";
}
if (!$errors)
{
echo "<p><b>Setting new incremental import time to: $str ($last_run)</b></p>\n";
}
}
catch (\Exception $e) {
echo "<p style'color: red'>".$e->getMessage()."</p>\n";
http_response_code(500);
exit;
}

View File

@ -449,6 +449,60 @@
<td colspan="2">&nbsp;</td>
</tr>
<tr class="th">
<td colspan="2"><b>{lang_Periodic_import_from_ADS_or_LDAP_into_EGroupware_database}:</b></td>
</tr>
<tr class="row_on">
<td>{lang_Source_(must_be_configured_above)}:</td>
<td>
<select name="newsettings[account_import_source]">
<option value="ads" {selected_account_import_source_ads}>ADS</option>
<option value="ldap" {selected_account_import_source_ldap}>LDAP</option>
</select>
</td>
</tr>
<tr class="row_off">
<td>{lang_What_to_import?}:</td>
<td>
<select name="newsettings[account_import_type]">
<option value="users" {selected_account_import_source_user}>{lang_just_users}</option>
<option value="users_groups" {selected_account_import_source_users_groups}>{lang_users,_groups_and_memberships}</option>
</select>
</td>
</tr>
<tr class="row_on">
<td>{lang_What_to_do_in_EGroupware_if_an_user_get_deleted?}:</td>
<td>
<select name="newsettings[account_import_delete]">
<option value="yes" {selected_account_import_delete_yes}>{lang_Delete_user_AND_his_data}</option>
<option value="deactivate" {selected_account_import_delete_deactivate}>{lang_Deactivate_user}</option>
<option value="no" {selected_account_import_delete_no}>{lang_Do_NOT_check_for_deleted_user}</option>
</select>
</td>
</tr>
<tr class="row_off">
<td>{lang_How_frequent_should_the_import_run?}:</td>
<td>
{lang_Every}
<input type="number" name="newsettings[account_import_frequency]" style="width: 3em" value="{value_account_import_frequency}"/>
{lang_hours_at}
<input type="time" name="newsettings[account_import_time]" value="{value_account_import_time}"/>
</td>
</tr>
<tr class="row_on">
<td>{lang_You_must_save_AND_run_an_inital_import,_before_the_periodic_import_will_start}:</td>
<td>
<button onclick="window.open('account_import.php?initial=true', '_blank')">{lang_Initial_import}</button>
<button onclick="window.open('account_import.php', '_blank')">{lang_Incremental_import}</button>
{lang_We_strongly_recomment_to_run_a_DB_backup_BEFORE_running_the_import!}
</td>
</tr>
<tr class="row_off">
<td colspan="2">&nbsp;</td>
</tr>
<tr class="th">
<td colspan="2"><b>{lang_If_using_Mail_authentication}:</b></td>
</tr>