DuplicateIndexBuilder::internalBuild

protected function internalBuild(array &$progressData)
{
	$offset = isset($progressData['OFFSET']) ? max((int)$progressData['OFFSET'], 0) : 0;
	$limit = isset($progressData['LIMIT']) ? max((int)$progressData['LIMIT'], 0) : 0;

	$dataSource = $this->getDataSource();
	$result = $dataSource->getList($offset, $limit);

	$rankings = $result->getAllRankings();
	DuplicateEntityRanking::initializeBulk($rankings,
		array('CHECK_PERMISSIONS' => $this->isPermissionCheckEnabled(), 'USER_ID' => $this->getUserID())
	);

	$rootEntityIDs = array();
	$items = $result->getItems();
	foreach($items as $matchHash => $item)
	{
		$rootEntityInfo = array();
		if($this->tryResolveRootEntity($item, $matchHash, $rootEntityInfo))
		{
			$entityID = $rootEntityInfo['ENTITY_ID'];
			$rootEntityIDs[] = $entityID;
			$item->setRootEntityID($entityID);
		}
		else
		{
			$result->removeItem($matchHash);
		}
	}

	$sortParams = $this->prepareSortParams($rootEntityIDs);
	$effectiveItemCount = 0;

	$items = $result->getItems();
	foreach($items as $matchHash => $item)
	{
		$enableOverwrite = $item->getOption('enableOverwrite', true);
		if(
			!$enableOverwrite
			&& $this->duplicateIndexExists($this->getPrimaryKey($matchHash)))
		{
			if ($this->params->limitByDirtyIndexItems())
			{
				$this->markAsNotDirty($this->getPrimaryKey($matchHash));
			}
			continue;
		}

		$data = $this->prepareTableData($matchHash, $item, $sortParams, true);
		$data['STATUS_ID'] = DuplicateStatus::PENDING;
		$this->saveDuplicateIndexItem($data);
		$effectiveItemCount++;
	}
	if ($this->params->limitByDirtyIndexItems())
	{
		$this->processInvalidDirtyItems($result->getInvalidItems());
	}

	$processedItemCount = $result->getProcessedItemCount();
	$progressData['EFFECTIVE_ITEM_COUNT'] = $effectiveItemCount;
	$progressData['PROCESSED_ITEM_COUNT'] = $processedItemCount;
	$progressData['OFFSET'] = $offset + $processedItemCount;

	return $this->isInProgress($progressData);
}