| public static getSCUByInstanceType ( $serverType, $platform ) |
public static function getSCUByInstanceType($serverType, $platform)
{
$scu = [];
$scu[SERVER_PLATFORMS::EC2] = array('t1.micro' => 0.25, 't2.nano' => 0.1, 't2.micro' => 0.25, 't2.small' => 0.5, 't2.medium' => 0.75, 't2.large' => 1, 'c1.medium' => 1.75, 'c1.xlarge' => 7.25, 'c3.large' => 2, 'c3.xlarge' => 4, 'c3.2xlarge' => 8, 'c3.4xlarge' => 16, 'c3.8xlarge' => 32, 'c4.large' => 2.25, 'c4.xlarge' => 4.5, 'c4.2xlarge' => 9, 'c4.4xlarge' => 17, 'c4.8xlarge' => 33, 'r3.large' => 2, 'r3.xlarge' => 4, 'r3.2xlarge' => 8, 'r3.4xlarge' => 16, 'r3.8xlarge' => 32, 'm1.small' => 0.75, 'm1.medium' => 1.5, 'm1.large' => 3.25, 'm1.xlarge' => 6.25, 'm2.xlarge' => 5.5, 'm2.2xlarge' => 11, 'm2.4xlarge' => 20, 'm3.medium' => 1.5, 'm3.large' => 3, 'm3.xlarge' => 6, 'm3.2xlarge' => 12, 'm4.large' => 3, 'm4.xlarge' => 6, 'm4.2xlarge' => 12, 'm4.4xlarge' => 16, 'm4.10xlarge' => 44, 'cc1.4xlarge' => 16, 'cc2.8xlarge' => 30, 'cr1.8xlarge' => 44, 'cg1.4xlarge' => 26, 'hi1.4xlarge' => 38, 'hs1.8xlarge' => 58, 'g2.2xlarge' => 8, 'g2.8xlarge' => 32, 'i2.xlarge' => 4, 'i2.2xlarge' => 8, 'i2.4xlarge' => 16, 'i2.8xlarge' => 32, 'd2.xlarge' => 4, 'd2.2xlarge' => 8, 'd2.4xlarge' => 16, 'd2.8xlarge' => 32);
$scu[SERVER_PLATFORMS::RACKSPACENG_US] = array('1' => 0.25, '2' => 0.75, '3' => 1.5, '4' => 3, '5' => 6, '6' => 12, '7' => 15, 'general1-1' => 0.25, 'general1-2' => 0.5, 'general1-4' => 1, 'general1-8' => 2, 'io1-15' => 3, 'io1-30' => 6, 'io1-60' => 12, 'io1-90' => 24, 'io1-120' => 48, 'performance1-1' => 0.5, 'performance1-2' => 1, 'performance1-4' => 2, 'performance1-8' => 4, 'performance2-15' => 8, 'performance2-30' => 16, 'performance2-60' => 32, 'performance2-90' => 48, 'performance2-120' => 64);
$scu[SERVER_PLATFORMS::RACKSPACENG_UK] = array('1' => 0.25, '2' => 0.75, '3' => 1.5, '4' => 3, '5' => 6, '6' => 12, '7' => 15, 'general1-1' => 0.25, 'general1-2' => 0.5, 'general1-4' => 1, 'general1-8' => 2, 'io1-15' => 3, 'io1-30' => 6, 'io1-60' => 12, 'io1-90' => 24, 'io1-120' => 48, 'performance1-1' => 0.5, 'performance1-2' => 1, 'performance1-4' => 2, 'performance1-8' => 4, 'performance2-15' => 8, 'performance2-30' => 16, 'performance2-60' => 32, 'performance2-90' => 48, 'performance2-120' => 64);
$scu[SERVER_PLATFORMS::GCE] = array('n1-standard-1-d' => 1.75, 'n1-standard-2-d' => 3.5, 'n1-standard-4-d' => 7, 'n1-standard-8-d' => 14, 'n1-standard-1' => 1.5, 'n1-standard-2' => 3, 'n1-standard-4' => 6, 'n1-standard-8' => 12, 'n1-standard-16' => 24, 'n1-highmem-2-d' => 4, 'n1-highmem-4-d' => 8, 'n1-highmem-8-d' => 16, 'n1-highmem-2' => 3.25, 'n1-highmem-4' => 6.5, 'n1-highmem-8' => 13, 'n1-highmem-16' => 26, 'n1-highcpu-2-d' => 2, 'n1-highcpu-4-d' => 4, 'n1-highcpu-8-d' => 8, 'n1-highcpu-2' => 1.75, 'n1-highcpu-4' => 3.5, 'n1-highcpu-8' => 7, 'n1-highcpu-16' => 14, 'f1-micro' => 0.25, 'g1-small' => 0.5);
if (!isset($scu[$platform][$serverType])) {
if (isset($scu[$platform])) {
// Check SCU only for defined clouds for private clouds, we don't care
trigger_error(sprintf('SCU has not been defined for instance type: "%s" on "%s" platform.', $serverType, $platform), E_USER_WARNING);
}
return 0;
}
return $scu[$platform][$serverType];
} /**
* {@inheritdoc}
* @see Scalr_System_Cronjob_MultiProcess_DefaultWorker::handleWork()
*/
function handleWork($farmId)
{
$this->cleanup();
$DBFarm = DBFarm::LoadByID($farmId);
$account = Scalr_Account::init()->loadById($DBFarm->ClientID);
$payAsYouGoTime = $account->getSetting(Scalr_Account::SETTING_BILLING_PAY_AS_YOU_GO_DATE);
$GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid() . $farmId));
$GLOBALS["LOGGER_FARMID"] = $farmId;
$this->logger->info("[" . $GLOBALS["SUB_TRANSACTIONID"] . "] Begin polling farm (ID: {$DBFarm->ID}, Name: {$DBFarm->Name}, Status: {$DBFarm->Status})");
// Collect information from database
$servers_count = $this->db->GetOne("\n SELECT COUNT(*) FROM servers WHERE farm_id = ? AND status NOT IN (?,?)\n ", array($DBFarm->ID, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED));
$this->logger->info("[FarmID: {$DBFarm->ID}] Found {$servers_count} farm instances in database");
if ($DBFarm->Status == FARM_STATUS::TERMINATED && $servers_count == 0) {
return;
}
foreach ($DBFarm->GetServersByFilter(array(), array('status' => SERVER_STATUS::PENDING_LAUNCH)) as $DBServer) {
/** @var DBServer $DBServer */
try {
if ($DBServer->cloudLocation) {
try {
Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: {$DBServer->cloudLocation} ({$DBServer->serverId})"));
$p = PlatformFactory::NewPlatform($DBServer->platform);
$list = $p->GetServersList($DBServer->GetEnvironmentObject(), $DBServer->cloudLocation);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->info(sprintf("Initializing cloud cache: FAILED"));
}
}
if ($DBServer->status != SERVER_STATUS::PENDING && $DBServer->status != SERVER_STATUS::PENDING_TERMINATE) {
if (!$p->IsServerExists($DBServer)) {
try {
$serverInfo = $p->GetServerExtendedInformation($DBServer);
} catch (Exception $e) {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] Crash check for server '%s' failed: %s", $DBFarm->ID, $DBServer->serverId, $e->getMessage()));
}
if (!$serverInfo) {
if ($p->debugLog) {
Logger::getLogger('Openstack')->fatal($p->debugLog);
}
if (!in_array($DBServer->status, array(SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::TERMINATED, SERVER_STATUS::SUSPENDED))) {
if ($DBServer->GetProperty(SERVER_PROPERTIES::CRASHED) == 1) {
if (PlatformFactory::isOpenstack($DBServer->platform)) {
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, 1);
} else {
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
Scalr::FireEvent($DBFarm->ID, new HostCrashEvent($DBServer));
}
} else {
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::CRASHED => 1, SERVER_PROPERTIES::MISSING => 1]);
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' found in database but not found on %s. Crashed.", $DBServer->serverId, $DBServer->platform)));
}
continue;
}
} else {
Logger::getLogger(LOG_CATEGORY::FARM)->error(sprintf("[CRASH][FarmID: %d] False-positive crash check: %s (EnvID: %d)", $DBFarm->ID, $DBServer->serverId, $DBServer->envId));
}
} else {
$DBServer->SetProperty(SERVER_PROPERTIES::CRASHED, "0");
$DBServer->SetProperty(SERVER_PROPERTIES::MISSING, "0");
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "AWS was not able to validate the provided access credentials") || stristr($e->getMessage(), "Unable to sign AWS API request. Please, check your X.509")) {
$env = Scalr_Environment::init()->LoadById($DBFarm->EnvID);
$env->status = Scalr_Environment::STATUS_INACTIVE;
$env->save();
$env->setPlatformConfig(array('system.auto-disable-reason' => $e->getMessage()), false);
return;
}
if (stristr($e->getMessage(), "Could not connect to host")) {
continue;
}
print "[0][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
continue;
}
/*
try {
$realStatus = $DBServer->GetRealStatus()->getName();
if ($realStatus == 'stopped') {
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, $realStatus);
continue;
} else {
if ($DBServer->GetProperty(SERVER_PROPERTIES::SUB_STATUS) == 'stopped')
$DBServer->SetProperty(SERVER_PROPERTIES::SUB_STATUS, "");
}
} catch (Exception $e) {}
*/
try {
if (!in_array($DBServer->status, array(SERVER_STATUS::SUSPENDED, SERVER_STATUS::TERMINATED, SERVER_STATUS::PENDING_TERMINATE, SERVER_STATUS::PENDING_SUSPEND))) {
$openstackErrorState = false;
if (PlatformFactory::isOpenstack($DBServer->platform)) {
if ($DBServer->GetRealStatus()->getName() === 'ERROR') {
$openstackErrorState = true;
}
}
if ($DBServer->GetRealStatus()->isTerminated() || $openstackErrorState) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0]);
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} elseif ($DBServer->GetRealStatus()->isSuspended()) {
// if server was suspended when it was running
if ($DBServer->status == SERVER_STATUS::RUNNING) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' (Platform: %s) not running (Real state: %s, Scalr status: %s).", $DBServer->serverId, $DBServer->platform, $DBServer->GetRealStatus()->getName(), $DBServer->status)));
$DBServer->SetProperties([SERVER_PROPERTIES::REBOOTING => 0, SERVER_PROPERTIES::RESUMING => 0, SERVER_PROPERTIES::SUB_STATUS => ""]);
$DBServer->remoteIp = "";
$DBServer->localIp = "";
$DBServer->status = SERVER_STATUS::SUSPENDED;
$DBServer->Save();
Scalr::FireEvent($DBFarm->ID, new HostDownEvent($DBServer));
continue;
} else {
// If server was suspended during initialization - we do not support this and need to terminate this instance
$DBServer->terminate(DBServer::TERMINATE_REASON_CRASHED);
continue;
}
}
}
if ($DBServer->status != SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->IsRunning()) {
if ($DBServer->status == SERVER_STATUS::SUSPENDED) {
//TODO: Depends on resume strategy need to set server status
// For Openstack we need to re-accociate IPs
$DBServer->SetProperty(\SERVER_PROPERTIES::RESUMING, 0);
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]);
}
}
Scalr::FireEvent($DBFarm->ID, new HostUpEvent($DBServer, ""));
continue;
} elseif (!in_array($DBServer->status, array(SERVER_STATUS::TERMINATED, SERVER_STATUS::TROUBLESHOOTING))) {
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
if ($DBServer->status == SERVER_STATUS::PENDING && $DBFarm->GetSetting(DBFarm::SETTING_EC2_VPC_ID)) {
if ($DBServer->GetFarmRoleObject()->GetSetting(DBFarmRole::SETTING_AWS_VPC_INTERNET_ACCESS) != 'outbound-only') {
$ipAddress = \Scalr\Modules\Platforms\Ec2\Helpers\EipHelper::setEipForServer($DBServer);
if ($ipAddress) {
$DBServer->remoteIp = $ipAddress;
$DBServer->Save();
}
}
}
}
try {
if ($DBServer->isOpenstack()) {
$this->openstackSetFloatingIp($DBServer);
}
} catch (Exception $e) {
if (!$DBServer->GetProperty(SERVER_PROPERTIES::SZR_IS_INIT_FAILED)) {
$DBServer->SetProperties([SERVER_PROPERTIES::SZR_IS_INIT_FAILED => 1, SERVER_PROPERTIES::SZR_IS_INIT_ERROR_MSG => $e->getMessage()]);
}
}
if ($DBServer->isCloudstack()) {
if ($DBServer->status == SERVER_STATUS::PENDING) {
$jobId = $DBServer->GetProperty(CLOUDSTACK_SERVER_PROPERTIES::LAUNCH_JOB_ID);
try {
$cs = $DBServer->GetEnvironmentObject()->cloudstack($DBServer->platform);
$res = $cs->queryAsyncJobResult($jobId);
if ($res->jobstatus == 1) {
$DBServer->SetProperties([CLOUDSTACK_SERVER_PROPERTIES::TMP_PASSWORD => $res->virtualmachine->password, CLOUDSTACK_SERVER_PROPERTIES::SERVER_NAME => $res->virtualmachine->name]);
}
//TODO: handle failed job: $res->jobresult->jobstatus == 2
} catch (Exception $e) {
if ($DBServer->farmId) {
Logger::getLogger("CloudStack")->error(new FarmLogMessage($DBServer->farmId, $e->getMessage()));
}
}
}
}
try {
$dtadded = strtotime($DBServer->dateAdded);
$DBFarmRole = $DBServer->GetFarmRoleObject();
$launch_timeout = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) > 0 ? $DBFarmRole->GetSetting(DBFarmRole::SETTING_SYSTEM_LAUNCH_TIMEOUT) : 900;
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
$DBServer->terminate(DBServer::TERMINATE_REASON_ROLE_REMOVED);
}
}
$scripting_event = false;
if ($DBServer->status == SERVER_STATUS::PENDING) {
$event = "hostInit";
$scripting_event = EVENT_TYPE::HOST_INIT;
} elseif ($DBServer->status == SERVER_STATUS::INIT) {
$event = "hostUp";
$scripting_event = EVENT_TYPE::HOST_UP;
}
if ($scripting_event && $dtadded) {
$scripting_timeout = (int) $this->db->GetOne("\n SELECT sum(timeout)\n FROM farm_role_scripts\n WHERE event_name=? AND\n farm_roleid=? AND issync='1'\n ", array($scripting_event, $DBServer->farmRoleId));
if ($scripting_timeout) {
$launch_timeout = $launch_timeout + $scripting_timeout;
}
if ($dtadded + $launch_timeout < time() && !$DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//Add entry to farm log
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("Server '%s' did not send '%s' event in %s seconds after launch (Try increasing timeouts in role settings). Considering it broken. Terminating instance.", $DBServer->serverId, $event, $launch_timeout)));
try {
$DBServer->terminate(array(DBServer::TERMINATE_REASON_SERVER_DID_NOT_SEND_EVENT, $event, $launch_timeout), false);
} catch (Exception $err) {
$this->logger->fatal($err->getMessage());
}
} elseif ($DBFarmRole->GetRoleObject()->hasBehavior(ROLE_BEHAVIORS::MONGODB)) {
//DO NOT TERMINATE MONGODB INSTANCES BY TIMEOUT! IT'S NOT SAFE
//THINK ABOUT WORKAROUND
}
}
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp && $DBServer->localIp != $ipaddresses['localIp']) {
Logger::getLogger(LOG_CATEGORY::FARM)->warn(new FarmLogMessage($DBFarm->ID, sprintf("RemoteIP: %s (%s), LocalIp: %s (%s) (Poller).", $DBServer->remoteIp, $ipaddresses['remoteIp'], $DBServer->localIp, $ipaddresses['localIp'])));
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
//TODO: Check health:
}
}
} elseif ($DBServer->status == SERVER_STATUS::RUNNING && $DBServer->GetRealStatus()->isRunning()) {
// Is IP address changed?
if (!$DBServer->IsRebooting()) {
$ipaddresses = PlatformFactory::NewPlatform($DBServer->platform)->GetServerIPAddresses($DBServer);
if ($ipaddresses['remoteIp'] && $DBServer->remoteIp != $ipaddresses['remoteIp'] || $ipaddresses['localIp'] && $DBServer->localIp != $ipaddresses['localIp']) {
Scalr::FireEvent($DBServer->farmId, new IPAddressChangedEvent($DBServer, $ipaddresses['remoteIp'], $ipaddresses['localIp']));
}
if ($payAsYouGoTime) {
$initTime = $DBServer->GetProperty(SERVER_PROPERTIES::INITIALIZED_TIME);
if ($initTime < $payAsYouGoTime) {
$initTime = $payAsYouGoTime;
}
$runningHours = ceil((time() - $initTime) / 3600);
$scuUsed = $runningHours * Scalr_Billing::getSCUByInstanceType($DBServer->GetFlavor(), $DBServer->platform);
$this->db->Execute("UPDATE servers_history SET scu_used = ?, scu_updated = 0 WHERE server_id = ?", array($scuUsed, $DBServer->serverId));
}
if ($DBServer->platform == SERVER_PLATFORMS::EC2) {
$env = Scalr_Environment::init()->loadById($DBServer->envId);
$ec2 = $env->aws($DBServer->GetCloudLocation())->ec2;
//TODO:
$time = $DBServer->GetProperty(EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME);
if (!$time || time() < $time + 1200) {
$isEnabled = $ec2->instance->describeAttribute($DBServer->GetCloudServerID(), InstanceAttributeType::disableApiTermination());
$DBServer->SetProperties([EC2_SERVER_PROPERTIES::IS_LOCKED => $isEnabled, EC2_SERVER_PROPERTIES::IS_LOCKED_LAST_CHECK_TIME => time()]);
}
}
} else {
//TODO: Check reboot timeout
}
}
} catch (Exception $e) {
if (stristr($e->getMessage(), "not found")) {
print $e->getTraceAsString() . "\n";
} elseif (stristr($e->getMessage(), "Request limit exceeded")) {
sleep(10);
print "[1:SLEEP][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
} else {
print "[1][Farm: {$farmId}] {$e->getMessage()} at {$e->getFile()}:{$e->getLine()}\n\n";
}
}
}
}