mirror of
https://github.com/EGroupware/egroupware.git
synced 2024-11-07 08:34:29 +01:00
* LDAP/AD/Setup: periodic import disables (add|edit)account hooks, if account-repository is not SQL and further performance improvements
- also log interactive initial or incremental import via setup - fix conflict resolution if account_id of user already used for a group
This commit is contained in:
parent
55116c7b49
commit
566dac47af
@ -35,6 +35,14 @@ class Import
|
|||||||
/** @var callable */
|
/** @var callable */
|
||||||
protected $_logger;
|
protected $_logger;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Conflict offset is added to an account_id/uidNumber, if it's already used (as gidNumber or by another user)
|
||||||
|
*/
|
||||||
|
const CONFLICT_OFFSET = 1000000;
|
||||||
|
/**
|
||||||
|
* Max. value for 32-bit signed integer columns like account_id
|
||||||
|
*/
|
||||||
|
const MAX_INTEGER = 2147483647;
|
||||||
/**
|
/**
|
||||||
* Filename => [attr, mask, regexp] for jpegphoto and pubkey attributes
|
* Filename => [attr, mask, regexp] for jpegphoto and pubkey attributes
|
||||||
*
|
*
|
||||||
@ -142,12 +150,31 @@ class Import
|
|||||||
|
|
||||||
* @return void
|
* @return void
|
||||||
*/
|
*/
|
||||||
protected function logger(string $message, string $level)
|
function logger(string $message, string $level)
|
||||||
{
|
{
|
||||||
if ($this->_logger)
|
if ($this->_logger)
|
||||||
{
|
{
|
||||||
call_user_func($this->_logger, $message, $level);
|
call_user_func($this->_logger, $message, $level);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// log to file too
|
||||||
|
$log = $GLOBALS['egw_info']['server']['files_dir'].'/'.self::LOG_FILE;
|
||||||
|
if (!file_exists($dir=dirname($log)) && !mkdir($dir) || !is_dir($dir) ||
|
||||||
|
!($fp = fopen($log, 'a+')))
|
||||||
|
{
|
||||||
|
if (!in_array($level, ['debug', 'detail']))
|
||||||
|
{
|
||||||
|
error_log(__METHOD__.' '.strtoupper($level).' '.$message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if ($this->_logger || !in_array($level, ['debug', 'detail']))
|
||||||
|
{
|
||||||
|
fwrite($fp, date('Y-m-d H:i:s O').' '.strtoupper($level).' '.$message."\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!empty($fp)) fclose($fp);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -219,6 +246,7 @@ class Import
|
|||||||
{
|
{
|
||||||
$filter[] = 'modified>='.$GLOBALS['egw_info']['server']['account_import_lastrun'];
|
$filter[] = 'modified>='.$GLOBALS['egw_info']['server']['account_import_lastrun'];
|
||||||
}
|
}
|
||||||
|
$num = 0;
|
||||||
$last_modified = null;
|
$last_modified = null;
|
||||||
$start_import = time();
|
$start_import = time();
|
||||||
$cookie = '';
|
$cookie = '';
|
||||||
@ -233,15 +261,19 @@ class Import
|
|||||||
$last_modified = $contact['modified'];
|
$last_modified = $contact['modified'];
|
||||||
}
|
}
|
||||||
$account = $this->accounts->read($contact['account_id']);
|
$account = $this->accounts->read($contact['account_id']);
|
||||||
$this->logger(json_encode($contact + $account), 'debug');
|
$this->logger(++$num.'. User: '.json_encode($contact + $account, JSON_UNESCAPED_SLASHES|JSON_UNESCAPED_UNICODE), 'debug');
|
||||||
// check if account exists in sql
|
// check if account exists in sql
|
||||||
if (!($account_id = $this->accounts_sql->name2id($account['account_lid'])))
|
if (!($account_id = $this->accounts_sql->name2id($account['account_lid'])))
|
||||||
{
|
{
|
||||||
$sql_account = $account;
|
$sql_account = $account;
|
||||||
// check if account_id is not yet taken by another user or group --> unset it to let DB assign a new one
|
// check if account_id is not yet taken by another user or group --> add offset or unset it to let DB assign a new one
|
||||||
if ($this->accounts_sql->read($account['account_id']))
|
while ($this->accounts_sql->read($sql_account['account_id']))
|
||||||
{
|
{
|
||||||
unset($sql_account['account_id']);
|
if (($sql_account['account_id'] += self::CONFLICT_OFFSET) > self::MAX_INTEGER)
|
||||||
|
{
|
||||||
|
unset($sql_account['account_id']);
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if ($dry_run)
|
if ($dry_run)
|
||||||
{
|
{
|
||||||
@ -250,9 +282,13 @@ class Import
|
|||||||
elseif (($account_id = $sql_account['account_id'] = $this->accounts_sql->save($sql_account, true)) > 0)
|
elseif (($account_id = $sql_account['account_id'] = $this->accounts_sql->save($sql_account, true)) > 0)
|
||||||
{
|
{
|
||||||
// run addaccount hook to create eg. home-directory or mail account
|
// run addaccount hook to create eg. home-directory or mail account
|
||||||
Api\Hooks::process($sql_account+array(
|
// only if account-repository is already SQL, NOT for migration
|
||||||
'location' => 'addaccount'
|
if (($GLOBALS['egw_info']['server']['account_repository'] ?? 'sql') === 'sql')
|
||||||
),False,True); // called for every app now, not only enabled ones)
|
{
|
||||||
|
Api\Hooks::process($sql_account+array(
|
||||||
|
'location' => 'addaccount'
|
||||||
|
),False,True); // called for every app now, not only enabled ones)
|
||||||
|
}
|
||||||
|
|
||||||
$this->logger("Successful created user '$account[account_lid]' (#$account[account_id]".
|
$this->logger("Successful created user '$account[account_lid]' (#$account[account_id]".
|
||||||
($account['account_id'] != $account_id ? " as #$account_id" : '').')', 'detail');
|
($account['account_id'] != $account_id ? " as #$account_id" : '').')', 'detail');
|
||||||
@ -266,7 +302,9 @@ class Import
|
|||||||
}
|
}
|
||||||
elseif ($account_id < 0)
|
elseif ($account_id < 0)
|
||||||
{
|
{
|
||||||
throw new \Exception("User '$account[account_lid]' already exists as group!");
|
$this->logger("User '$account[account_lid]' (#$account[account_id]) already exists as group --> NOT imported!", 'error');
|
||||||
|
$errors++;
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
elseif (!($sql_account = $this->accounts_sql->read($account_id)))
|
elseif (!($sql_account = $this->accounts_sql->read($account_id)))
|
||||||
{
|
{
|
||||||
@ -298,10 +336,13 @@ class Import
|
|||||||
if ($this->accounts_sql->save($to_update) > 0)
|
if ($this->accounts_sql->save($to_update) > 0)
|
||||||
{
|
{
|
||||||
// run editaccount hook to create eg. home-directory or mail account
|
// run editaccount hook to create eg. home-directory or mail account
|
||||||
Api\Hooks::process($to_update+array(
|
// only if account-repository is already SQL, NOT for migration
|
||||||
'location' => 'editaccount'
|
if (($GLOBALS['egw_info']['server']['account_repository'] ?? 'sql') === 'sql')
|
||||||
),False,True); // called for every app now, not only enabled ones)
|
{
|
||||||
|
Api\Hooks::process($to_update + array(
|
||||||
|
'location' => 'editaccount'
|
||||||
|
), False, True); // called for every app now, not only enabled ones)
|
||||||
|
}
|
||||||
$this->logger("Successful updated user '$account[account_lid]' (#$account_id): " .
|
$this->logger("Successful updated user '$account[account_lid]' (#$account_id): " .
|
||||||
json_encode($diff, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'detail');
|
json_encode($diff, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), 'detail');
|
||||||
if (!$new) $new = false;
|
if (!$new) $new = false;
|
||||||
@ -330,6 +371,7 @@ class Import
|
|||||||
{
|
{
|
||||||
$sql_contact = $contact;
|
$sql_contact = $contact;
|
||||||
unset($sql_contact['id']); // LDAP contact-id is the UID!
|
unset($sql_contact['id']); // LDAP contact-id is the UID!
|
||||||
|
$sql_contact['account_id'] = $account_id;
|
||||||
if (!$this->contacts_sql->save($sql_contact))
|
if (!$this->contacts_sql->save($sql_contact))
|
||||||
{
|
{
|
||||||
$sql_contact['id'] = $this->contacts_sql->data['id'];
|
$sql_contact['id'] = $this->contacts_sql->data['id'];
|
||||||
@ -367,6 +409,7 @@ class Import
|
|||||||
$to_update = array_merge($sql_contact, array_filter($contact, static function ($attr) {
|
$to_update = array_merge($sql_contact, array_filter($contact, static function ($attr) {
|
||||||
return $attr !== null && $attr !== '';
|
return $attr !== null && $attr !== '';
|
||||||
}));
|
}));
|
||||||
|
unset($to_update['account_id']); // no need to update, specially as account_id might be different!
|
||||||
$to_update['id'] = $sql_contact['id'];
|
$to_update['id'] = $sql_contact['id'];
|
||||||
if (($diff = array_diff_assoc($to_update, $sql_contact)))
|
if (($diff = array_diff_assoc($to_update, $sql_contact)))
|
||||||
{
|
{
|
||||||
@ -583,10 +626,10 @@ class Import
|
|||||||
$groups = $sql_groups;
|
$groups = $sql_groups;
|
||||||
}
|
}
|
||||||
|
|
||||||
$created = $updated = $uptodate = $errors = $deleted = 0;
|
$created = $updated = $uptodate = $errors = $deleted = $num = 0;
|
||||||
foreach($this->accounts->search(['type' => 'groups', 'modified' => $modified]) as $account_id => $group)
|
foreach($this->accounts->search(['type' => 'groups', 'modified' => $modified]) as $account_id => $group)
|
||||||
{
|
{
|
||||||
$this->logger(json_encode($group, JSON_UNESCAPED_UNICODE|JSON_UNESCAPED_SLASHES), 'debug');
|
$this->logger(++$num.'. Group: '.json_encode($group, JSON_UNESCAPED_UNICODE|JSON_UNESCAPED_SLASHES), 'debug');
|
||||||
|
|
||||||
if (!($sql_id = array_search($group['account_lid'], $sql_groups)))
|
if (!($sql_id = array_search($group['account_lid'], $sql_groups)))
|
||||||
{
|
{
|
||||||
@ -673,7 +716,11 @@ class Import
|
|||||||
$groups[$sql_id] = $group['account_lid'];
|
$groups[$sql_id] = $group['account_lid'];
|
||||||
|
|
||||||
// we need to record and return the id's to update members, AFTER users are created/updated
|
// we need to record and return the id's to update members, AFTER users are created/updated
|
||||||
$set_members[$sql_id] = $this->accounts->read($group['account_id'])['members'];
|
// only for incremental run, initial run set's memberships with the user anyway (more efficient for LDAP!)
|
||||||
|
if (!empty($modified))
|
||||||
|
{
|
||||||
|
$set_members[$sql_id] = $this->accounts->members($group['account_id']);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// delete the groups not returned from LDAP, groups can NOT be deactivated, we just delete them in the DB
|
// delete the groups not returned from LDAP, groups can NOT be deactivated, we just delete them in the DB
|
||||||
@ -826,43 +873,21 @@ class Import
|
|||||||
public static function async()
|
public static function async()
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
$log = $GLOBALS['egw_info']['server']['files_dir'].'/'.self::LOG_FILE;
|
$import = new self();
|
||||||
if (!file_exists($dir=dirname($log)) && !mkdir($dir) || !is_dir($dir) ||
|
$import->logger(date('Y-m-d H:i:s O').' LDAP account import started', 'info');
|
||||||
!($fp = fopen($log, 'a+')))
|
|
||||||
{
|
|
||||||
$logger = static function($str, $level)
|
|
||||||
{
|
|
||||||
if (!in_array($level, ['debug', 'detail']))
|
|
||||||
{
|
|
||||||
error_log(__METHOD__.' '.strtoupper($level).' '.$str);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
$logger = static function($str, $level) use ($fp)
|
|
||||||
{
|
|
||||||
if (!in_array($level, ['debug', 'detail']))
|
|
||||||
{
|
|
||||||
fwrite($fp, date('Y-m-d H:i:s O').' '.strtoupper($level).' '.$str."\n");
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
$logger(date('Y-m-d H:i:s O').' LDAP account import started', 'info');
|
|
||||||
$import = new self($logger);
|
|
||||||
$import->run(false);
|
$import->run(false);
|
||||||
$logger(date('Y-m-d H:i:s O').' LDAP account import finished'.(!empty($fp)?"\n":''), 'info');
|
$import->logger(date('Y-m-d H:i:s O').' LDAP account import finished'.(!empty($fp)?"\n":''), 'info');
|
||||||
}
|
}
|
||||||
catch (\InvalidArgumentException $e) {
|
catch (\InvalidArgumentException $e) {
|
||||||
_egw_log_exception($e);
|
_egw_log_exception($e);
|
||||||
// disable async job, something is not configured correct
|
// disable async job, something is not configured correct
|
||||||
self::installAsyncJob();
|
self::installAsyncJob();
|
||||||
$logger('Async job for periodic import canceled', 'fatal');
|
$import->logger('Async job for periodic import canceled', 'fatal');
|
||||||
}
|
}
|
||||||
catch (\Exception $e) {
|
catch (\Exception $e) {
|
||||||
_egw_log_exception($e);
|
_egw_log_exception($e);
|
||||||
|
$import->logger('Error: '.$e->getMessage());
|
||||||
}
|
}
|
||||||
if (!empty($fp)) fclose($fp);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -55,11 +55,6 @@ class Ldap
|
|||||||
*/
|
*/
|
||||||
var $ldapServerInfo;
|
var $ldapServerInfo;
|
||||||
|
|
||||||
/**
|
|
||||||
* @var int $ldapLimit how many rows to fetch from ldap server
|
|
||||||
*/
|
|
||||||
var $ldapLimit = 2000;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @var string $personalContactsDN holds the base DN for the personal addressbooks
|
* @var string $personalContactsDN holds the base DN for the personal addressbooks
|
||||||
*/
|
*/
|
||||||
@ -1277,11 +1272,11 @@ class Ldap
|
|||||||
|
|
||||||
if($_addressbooktype == self::ALL || $_ldapContext == $this->allContactsDN)
|
if($_addressbooktype == self::ALL || $_ldapContext == $this->allContactsDN)
|
||||||
{
|
{
|
||||||
$result = ldap_search($this->ds, $_ldapContext, $_filter, $_attributes, 0, $this->ldapLimit, null, null, $control);
|
$result = ldap_search($this->ds, $_ldapContext, $_filter, $_attributes, null, null, null, null, $control);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
$result = @ldap_list($this->ds, $_ldapContext, $_filter, $_attributes, 0, $this->ldapLimit, null, null, $control);
|
$result = ldap_list($this->ds, $_ldapContext, $_filter, $_attributes, null, null, null, null, $control);
|
||||||
}
|
}
|
||||||
if(!$result || !$entries = ldap_get_entries($this->ds, $result)) return array();
|
if(!$result || !$entries = ldap_get_entries($this->ds, $result)) return array();
|
||||||
$this->total += $entries['count'];
|
$this->total += $entries['count'];
|
||||||
@ -1295,9 +1290,9 @@ class Ldap
|
|||||||
$this->total = $serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count'];
|
$this->total = $serverctrls[LDAP_CONTROL_VLVRESPONSE]['value']['count'];
|
||||||
$start = null; // so caller does NOT run it's own limit
|
$start = null; // so caller does NOT run it's own limit
|
||||||
}
|
}
|
||||||
elseif (isset($serverctrls[LDAP_CONTROL_PAGEDRESULTS]['value']['cookie']))
|
else
|
||||||
{
|
{
|
||||||
$start[2] = $serverctrls[LDAP_CONTROL_PAGEDRESULTS]['value']['cookie'];
|
$start[2] = $serverctrls[LDAP_CONTROL_PAGEDRESULTS]['value']['cookie'] ?? '';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -50,6 +50,7 @@ try {
|
|||||||
$import->showLog();
|
$import->showLog();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
$import->logger("Manual import started via setup: initial=$_GET[initial], dry-run=$_GET[dry_run]", 'info');
|
||||||
$import->run(!empty($_GET['initial']) && $_GET['initial'] !== 'false',
|
$import->run(!empty($_GET['initial']) && $_GET['initial'] !== 'false',
|
||||||
!empty($_GET['dry_run'] ?? $_GET['dry-run']) && ($_GET['dry_run'] ?? $_GET['dry-run']) !== 'false');
|
!empty($_GET['dry_run'] ?? $_GET['dry-run']) && ($_GET['dry_run'] ?? $_GET['dry-run']) !== 'false');
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user