RobotsFile::load

protected function load()
{
	if($this->isExists() && !$this->bLoaded)
	{
		$contents = $this->getContents();
		$arLines = preg_split("/\n+/", $contents);
		$currentAgent = '';
		if(count($arLines) > 0)
		{
			$strSectionCompare = ToUpper(self::SECTION_RULE);
			foreach($arLines as $line)
			{
				$line = trim($line);

				if($line <> '')
				{
					$rule = $this->parseRule($line);
					if(ToUpper($rule[0]) == $strSectionCompare)
					{
						$currentAgent = $rule[1];
					}
					elseif ($currentAgent != '')
					{

						if(!is_array($this->contents[$currentAgent]))
						{
							$this->contents[$currentAgent] = array();
						}

						$this->contents[$currentAgent][] = $rule;
					}
				}
			}
		}
	}
	$this->bLoaded = true;
}