PhraseIndexCollection::collect

public function collect(TranslateFilter $filter = null, TranslateControllerITimeLimit $timer = null, TranslateFilter $seek = null): int
{
	if (isset($filter, $filter->path))
	{
		$relPath = $filter->path;
	}
	else
	{
		$relPath = TranslateConfig::getDefaultPath();
	}

	$relPath = '/'. trim($relPath, '/');
	$relPath = TranslateIOPath::replaceLangId($relPath, '#LANG_ID#');

	$checkLanguages = TranslateConfig::getEnabledLanguages();
	if (isset($filter, $filter->langId))
	{
		$checkLanguages = array_intersect($filter->langId, $checkLanguages);
	}

	$topPathRes = IndexInternalsPathIndexTable::getList([
		'select' => ['ID'],
		'filter' => ['=PATH' => $relPath]
	]);
	if (!($topPath = $topPathRes->fetch()))
	{
		return 0;
	}

	$fileFilter = [
		'=PATH.DESCENDANTS.PARENT_ID' => $topPath['ID'],//ancestor
		'=LANG_ID' => $checkLanguages,
		//todo: add filter by INDEXED_TIME
	];
	if (isset($seek, $seek->pathId))
	{
		$fileFilter['>PATH_ID'] = $seek->pathId;
	}

	MainApplication::getConnection()->queryExecute("SET SESSION group_concat_max_len = 100000");

	$fileListQuery = IndexInternalsFileIndexTable::query();

	$fileListQuery
		->addSelect('PATH_ID')

		->registerRuntimeField(new MainORMFieldsExpressionField('FILE_IDS', "GROUP_CONCAT(%s ORDER BY (%s) SEPARATOR '\n')", ['ID', 'ID']))
		->addSelect('FILE_IDS')

		->registerRuntimeField(new MainORMFieldsExpressionField('LANG_IDS', "GROUP_CONCAT(%s ORDER BY (%s) SEPARATOR '\n')", ['LANG_ID', 'ID']))
		->addSelect('LANG_IDS')

		->registerRuntimeField(new MainORMFieldsExpressionField('FULL_PATHS', "GROUP_CONCAT(%s ORDER BY (%s) SEPARATOR '\n')", ['FULL_PATH', 'ID']))
		->addSelect('FULL_PATHS')

		->setFilter($fileFilter)
		->setOrder(['PATH_ID' => 'ASC'])
		->addGroup('PATH_ID')
	;

	$fileListRes = $fileListQuery->exec();

	$phraseId = IndexInternalsPhraseIndexTable::query()
		->registerRuntimeField(new MainORMFieldsExpressionField('MAXID', 'MAX(%s)', ['ID']))
		->addSelect('MAXID')
		->exec()
		->fetch()['MAXID'];

	$processedItemCount = 0;

	while (true)
	{
		$lastPathId = null;
		$filePortion = [];
		while ($pathRow = $fileListRes->fetch())
		{
			$filePortion[] = $pathRow;
			if (count($filePortion) >= 5)
			{
				break;
			}
		}
		if (empty($filePortion))
		{
			break;
		}

		$fileData = [];
		$phraseCodeData = [];
		$phraseData = [];
		$pathIdPortion = [];
		$nonexistentFiles = [];

		foreach ($filePortion as $indexFile)
		{
			$pathId = (int)$indexFile['PATH_ID'];
			$pathIdPortion[] = $lastPathId = $pathId;

			$fileIds = [];
			foreach (explode("n", $indexFile['FILE_IDS']) as $v)
			{
				$fileIds[] = (int)$v;
			}

			$langIds = [];
			foreach (explode("n", $indexFile['LANG_IDS']) as $v)
			{
				$langIds[] = trim($v);
			}

			$filePaths = [];
			foreach (explode("n", $indexFile['FULL_PATHS']) as $v)
			{
				$filePaths[] = trim($v);
			}

			foreach ($fileIds as $inx => $indexFileId)
			{
				$langId = $langIds[$inx];
				$fullPath = $filePaths[$inx];

				if (self::$verbose)
				{
					echo "Lang file: {$fullPath}n";
				}

				$current = new TranslateFile($fullPath);
				$current->setLangId($langId);
				if (!$current->load())
				{
					$nonexistentFiles[] = $indexFileId;
					continue;
				}

				$fileData[] = [
					'ID' => $indexFileId,
					'PATH_ID' => $pathId,
					'LANG_ID' => $langId,
					'PHRASE_COUNT' => $current->count(),
					'FULL_PATH' => $current->getPath(),
					'INDEXED' => 'Y',
					'INDEXED_TIME' => new MainTypeDateTime(),
				];

				foreach ($current as $code => $phrase)
				{
					$phraseId ++;
					$phraseCodeData[] = [
						'ID' => $phraseId,
						'FILE_ID' => $indexFileId,
						'PATH_ID' => $pathId,
						'LANG_ID' => $langId,
						'CODE' => $code,
					];
					if (!isset($phraseData[$langId]))
					{
						$phraseData[$langId] = [];
					}
					$phraseData[$langId][] = [
						'ID' => $phraseId,
						'FILE_ID' => $indexFileId,
						'PATH_ID' => $pathId,
						'CODE' => $code,
						'PHRASE' => $phrase,
					];
				}
			}

			$processedItemCount += count($fileIds);
		}

		// delete
		IndexInternalsPhraseIndexTable::bulkDelete(['=PATH_ID' => $pathIdPortion, '=LANG_ID' => $checkLanguages]);

		if (count($nonexistentFiles) > 0)
		{
			IndexInternalsPhraseIndexTable::bulkDelete(['=FILE_ID' => $nonexistentFiles]);
			IndexInternalsFileIndexTable::bulkDelete(['=ID' => $nonexistentFiles]);
		}
		foreach (TranslateConfig::getLanguages() as $langId)
		{
			$ftsClass = IndexInternalsPhraseFts::getFtsEntityClass($langId);
			$ftsClass::bulkDelete(['=FILE_ID' => $nonexistentFiles]);
		}

		// Add
		if (count($phraseCodeData) > 0)
		{
			IndexInternalsFileIndexTable::bulkAdd($fileData, 'ID');
			IndexInternalsPhraseIndexTable::bulkAdd($phraseCodeData);
		}
		foreach ($phraseData as $langId => $phraseLangData)
		{
			$ftsClass = IndexInternalsPhraseFts::getFtsEntityClass($langId);
			$ftsClass::bulkAdd($phraseLangData, 'ID');
		}

		IndexInternalsPathIndexTable::bulkUpdate(
			['INDEXED' => 'Y', 'INDEXED_TIME' => new MainTypeDateTime()],
			['=ID' => $pathIdPortion]
		);

		if ($timer !== null && $timer->hasTimeLimitReached())
		{
			if ($seek !== null)
			{
				$seek->nextPathId = $lastPathId;
			}
			break;
		}
	}


	if ($timer === null || !$timer->hasTimeLimitReached())
	{
		IndexInternalsPathIndexTable::bulkUpdate(
			['INDEXED' => 'Y', 'INDEXED_TIME' => new MainTypeDateTime()],
			[
				'=IS_DIR' => 'Y',
				'=DESCENDANTS.PARENT_ID' => $topPath['ID'],//ancestor
			]
		);
	}

	return $processedItemCount;
}