RebuildUserDuplicateIndexAgent::rebuildIndex

protected function rebuildIndex(AutoSearchUserSettings $userSettings): bool
{
	$progressData = $userSettings->getProgressData();
	$userId = $userSettings->getUserId();
	$entityTypeId = $userSettings->getEntityTypeId();

	$dedupeConfig = $this->getDedupeConfig($entityTypeId, $userId);
	$enablePermissionCheck = !\CCrmPerms::IsAdmin($userId);
	$isStart = ($userSettings->getStatusId() === AutoSearchUserSettings::STATUS_NEW);
	if ($isStart)
	{
		$userSettings
			->setStatusId(AutoSearchUserSettings::STATUS_INDEX_REBUILDING)
			->save()
		;

		// types and scope used in previous run:
		$prevTypeIds = $progressData['TYPE_IDS'] ?? [];
		$prevScope = $progressData['CURRENT_SCOPE'] ?? DuplicateIndexType::DEFAULT_SCOPE;

		$progressData = [
			'PREV_TYPE_IDS' => $prevTypeIds,
			'PREV_SCOPE' => $prevScope,
			'TYPE_IDS' => $dedupeConfig['typeIDs'],
			'CURRENT_SCOPE' => $dedupeConfig['scope'],
			'CURRENT_TYPE_INDEX' => 0,
			'PROCESSED_ITEMS' => 0,
			'FOUND_CHANGED_ITEMS' => 0,
			'FOUND_ITEMS' => 0,
			'TOTAL_ENTITIES' => 0,
			'STARTED_TIMESTAMP' => time(),
		];

		$effectiveTypeIDs = $progressData['TYPE_IDS'];
		$effectiveScope = $progressData['CURRENT_SCOPE'];
		$currentTypeIndex = $progressData['CURRENT_TYPE_INDEX'];
	}
	else
	{
		$effectiveTypeIDs = $progressData['TYPE_IDS'] ?? null;
		if (!is_array($effectiveTypeIDs) || empty($effectiveTypeIDs))
		{
			$effectiveTypeIDs = $dedupeConfig['typeIDs'];
		}
		$effectiveScope = $progressData['CURRENT_SCOPE'] ?? DuplicateIndexType::DEFAULT_SCOPE;
		$currentTypeIndex = isset($progressData['CURRENT_TYPE_INDEX'])
			? (int)$progressData['CURRENT_TYPE_INDEX'] : 0;
	}

	$effectiveTypeQty = count($effectiveTypeIDs);
	if ($currentTypeIndex >= $effectiveTypeQty)
	{
		$userSettings->calcAndSetNextExecTime();
		$userSettings->setStatusId(AutoSearchUserSettings::STATUS_NEW);
		$userSettings->save();

		return false;
	}

	// fast algorithm can be used only
	// if data for this typeId was already processed at least once
	// and scope was not changed:
	$checkChangedOnly =
		$userSettings->getCheckChangedOnly() &&
		$progressData['PREV_SCOPE'] == $progressData['CURRENT_SCOPE'] &&
		in_array($effectiveTypeIDs[$currentTypeIndex], $progressData['PREV_TYPE_IDS']);

	$builder = DuplicateManager::createAutomaticIndexBuilder(
		$effectiveTypeIDs[$currentTypeIndex],
		$entityTypeId,
		$userId,
		$enablePermissionCheck,
		[
			'SCOPE' => $effectiveScope,
			'LAST_INDEX_DATE' => $userSettings->getLastExecTime(),
			'CHECK_CHANGED_ONLY' => $checkChangedOnly,
		]
	);

	$buildData = $progressData['BUILD_DATA'] ?? [];

	$offset = (int)($buildData['OFFSET'] ?? 0);
	if ($offset === 0)
	{
		$builder->remove();
	}

	$limit = (int)($buildData['LIMIT'] ?? 0);
	if ($limit === 0)
	{
		$buildData['LIMIT'] = $this->getRebuildIndexLimit();
	}

	$isInProgress = $builder->build($buildData);
	if (isset($buildData['PROCESSED_ITEM_COUNT']))
	{
		$progressData['PROCESSED_ITEMS'] = (int)($progressData['PROCESSED_ITEMS'] ?? 0);
		$progressData['PROCESSED_ITEMS'] += $buildData['PROCESSED_ITEM_COUNT'];
	}

	if (isset($buildData['EFFECTIVE_ITEM_COUNT']))
	{
		$progressData['FOUND_CHANGED_ITEMS'] = (int)($progressData['FOUND_CHANGED_ITEMS'] ?? 0);
		$progressData['FOUND_CHANGED_ITEMS'] += $buildData['EFFECTIVE_ITEM_COUNT'];
	}

	$progressData['BUILD_DATA'] = $buildData;

	$isFinal = false;
	if (!$isInProgress)
	{
		$isFinal = $currentTypeIndex === ($effectiveTypeQty - 1);
		if (!$isFinal)
		{
			$progressData['CURRENT_TYPE_INDEX'] = ++$currentTypeIndex;
			unset($progressData['BUILD_DATA']);
		}
	}

	if ($isFinal)
	{
		if ($userSettings->getCheckChangedOnly())
		{
			// if search settings was changed,
			// possibly there are indexes for invalid types or scopes
			// and necessary to remove them:
			$unusedTypeIds = array_diff($progressData['PREV_TYPE_IDS'], $progressData['TYPE_IDS']);
			if (count($unusedTypeIds))
			{
				$builder->removeUnusedIndexByTypeIds($unusedTypeIds);
			}
			if ($progressData['PREV_SCOPE'] != $progressData['CURRENT_SCOPE'])
			{
				$builder->removeUnusedIndexByScope($progressData['PREV_SCOPE']);
			}
		}

		$foundItems = AutomaticDuplicateList::getTotalItems(
			$userId,
			$entityTypeId,
			$effectiveTypeIDs,
			$effectiveScope
		);

		$isMergeEnabled = $userSettings->getIsMergeEnabled();
		if ($foundItems > 0)
		{
			$totalEntries = AutomaticDuplicateList::getTotalEntityCount(
				$userId,
				$entityTypeId,
				$effectiveTypeIDs,
				$effectiveScope
			);

			$userSettings->setStatusId(
				$isMergeEnabled ?
					AutoSearchUserSettings::STATUS_MERGING :
					AutoSearchUserSettings::STATUS_READY_TO_MERGE
			);
		}
		else
		{
			$totalEntries = 0;
			$userSettings->setStatusId(AutoSearchUserSettings::STATUS_NEW);
		}
		$userSettings->calcAndSetNextExecTime();

		$userSettings->setLastExecTime(
			DateTime::createFromTimestamp($progressData['STARTED_TIMESTAMP'] ?? time())
		);
		$userSettings->setCheckChangedOnly(true);

		if ($totalEntries > 0 && Loader::includeModule('pull'))
		{
			\Bitrix\Pull\Event::add($userId, [
				'module_id' => 'crm',
				'command' => 'dedupe.autosearch.startMerge',
				'params' => $isMergeEnabled ?
					[
						'status' => 'MERGING',
						'entityTypeId' => $entityTypeId,
					] :
					[
						'status' => 'READY_TO_MERGE',
						'entityTypeId' => $entityTypeId,
						'progressData' => [
							'TOTAL_ENTITIES' => $totalEntries,
							'FOUND_ITEMS' => $foundItems,
						],
					],
			]);
		}

		$progressData['TOTAL_ENTITIES'] = $totalEntries;
		$progressData['FOUND_ITEMS'] = $foundItems;
	}

	$userSettings->setProgressData($progressData);
	$userSettings->save();

	return !$isFinal;
}