ECS Blue/Green デプロイでもリクエストカウント追跡のオートスケーリングを利用したい

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

{
  "ScalingPolicies": [
    {
      //    :
      //  （略）
      //    :
      "TargetTrackingScalingPolicyConfiguration": {
          "TargetValue": 10.0,
          "PredefinedMetricSpecification": {
              "PredefinedMetricType": "ALBRequestCountPerTarget",
              "ResourceLabel": "app/<Alb_Name>/<Alb_Id>/targetgroup/<Target_Group_1_Name>/<Target_Group_1_Id>"
          },
          //    :
          //  （略）
          //    :
      },
      "Alarms": [
        {
            "AlarmName": "TargetTracking-service/<Cluster_Name>/<Service_Name>-AlarmHigh-XXXXXXXXXX",
            "AlarmARN": "arn:aws:cloudwatch:<Region>:<AWS_Account>:alarm:TargetTracking-service/<Cluster_Name>/<Service_Name>-AlarmHigh-XXXXXXXXXX"
        },
        {
            "AlarmName": "TargetTracking-service/<Cluster_Name>/<Service_Name>-AlarmLow-YYYYYYYYYY",
            "AlarmARN": "arn:aws:cloudwatch:<Region>:<AWS_Account>:alarm:TargetTracking-service/<Cluster_Name>/<Service_Name>-AlarmLow-YYYYYYYYYY"
        }
      ],
      "CreationTime": "2021-08-22T22:23:19.715000+09:00"
    }
  ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

version: 0.0
Resources:
  - TargetService:
      Type: AWS::ECS::Service
      Properties:
        TaskDefinition: <TASK_DEFINITION>
        LoadBalancerInfo:
          ContainerName: "app"
          ContainerPort: 8080
Hooks:
  - BeforeInstall: BeforeInstall-Lambda
  - AfterAllowTraffic: AfterAllowTraffic-Lambda

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

import json
import logging
import os
import sys
import boto3
from botocore.exceptions import ClientError

POLICY_NAME = os.getenv('POLICY_NAME')
ECS_CLUSTER_NAME = os.getenv('ECS_CLUSTER_NAME')
ECS_SERVICE_NAME = os.getenv('ECS_SERVICE_NAME')

logger = logging.getLogger(__name__)
logger.setLevel(os.getenv('LOG_LEVEL', 'WARNING'))
formatter = logging.Formatter("%(asctime)s %(name)s:%(lineno)s [%(levelname)s] %(funcName)s : %(message)s", "%Y-%m-%dT%H:%M:%S%z")

for handler in logger.handlers:
  handler.setFormatter(formatter)

codedeploy = boto3.client('codedeploy')
appAutoScaling = boto3.client('application-autoscaling')

def lambda_handler(event, context):

    deletePolicy()

    if 'DeploymentId' in event and 'LifecycleEventHookExecutionId' in event:
        logger.info('Invoked by CodeDeploy')
        res = codedeploy.put_lifecycle_event_hook_execution_status(
            deploymentId = event['DeploymentId'],
            lifecycleEventHookExecutionId = event['LifecycleEventHookExecutionId'],
            status = 'Succeeded'
        )
        logger.debug(f"codedeploy.put_lifecycle_event_hook_execution_status() response: {res}")

    return {
        'statusCode': 200,
        'body': json.dumps(res)
    }

def deletePolicy():
    try:
        res = appAutoScaling.delete_scaling_policy(
            PolicyName = POLICY_NAME,
            ServiceNamespace = 'ecs',
            ResourceId = f"service/{ECS_CLUSTER_NAME}/{ECS_SERVICE_NAME}",
            ScalableDimension = 'ecs:service:DesiredCount'
        )
        logger.debug(f"appAutoScaling.delete_scaling_policy() response: {res}")
    except ClientError as e:
        res = { 'error': e }
        logger.error(e)

    return res

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

import json
import logging
import os
import re
import sys
import boto3
from botocore.exceptions import ClientError

POLICY_NAME = os.getenv('POLICY_NAME', '')
ECS_CLUSTER_NAME = os.getenv('ECS_CLUSTER_NAME', '')
ECS_SERVICE_NAME = os.getenv('ECS_SERVICE_NAME', '')
ALB_NAME = os.getenv('ALB_NAME', '')
TARGET_GROUP_NAME_1 = os.getenv('TARGET_GROUP_NAME_1', '')
TARGET_GROUP_NAME_2 = os.getenv('TARGET_GROUP_NAME_2', '')
REQUEST_COUNT_TARGET_VALUE = float(os.getenv('REQUEST_COUNT_TARGET_VALUE', '100.0'))
SCALE_OUT_COOL_DOWN = int(os.getenv('SCALE_OUT_COOL_DOWN', '300'))
SCALE_IN_COOL_DOWN = int(os.getenv('SCALE_IN_COOL_DOWN', '300'))
DISABLE_SCALE_IN = os.getenv('DISABLE_SCALE_IN', 'False')
DISABLE_SCALE_IN = False if DISABLE_SCALE_IN == 'False' else bool(DISABLE_SCALE_IN)

logger = logging.getLogger(__name__)
logger.setLevel(os.getenv('LOG_LEVEL', 'WARNING'))
formatter = logging.Formatter("%(asctime)s %(name)s:%(lineno)s [%(levelname)s] %(funcName)s : %(message)s", "%Y-%m-%dT%H:%M:%S%z")

for handler in logger.handlers:
    handler.setFormatter(formatter)

codedeploy = boto3.client('codedeploy')
appAutoScaling = boto3.client('application-autoscaling')
elbv2 = boto3.client('elbv2')

class AutoScalingPolicy:

    re_elb_arn = re.compile(r'arn:aws(-[^:]+)?:elasticloadbalancing:[^:]+:\d{12}:loadbalancer/(.+)$')
    re_targetGroup_arn = re.compile(r'arn:aws(-[^:]+)?:elasticloadbalancing:[^:]+:\d{12}:(targetgroup/.+)$')

    def __init__(self) -> None:
        self.logger = logger.getChild('AppTask')
        pass

    def get_load_balancer_arn(self, name):
        res = elbv2.describe_load_balancers(Names = [name])
        self.logger.debug(f"elbv2.describe_load_balancers() response: {res}")

        arn = res['LoadBalancers'][0]['LoadBalancerArn']
        self.logger.debug(f"Load Balancer ARN: {arn}")
        return arn

    def parse_resource_label_of_elb(self, arn):
        m = self.re_elb_arn.match(arn)
        self.logger.debug(f"Resource Lable of ELB: {m.group(2)}")
        return m.group(2)

    def get_current_target_group_arn(self, tgNames, elbArn):
        res = elbv2.describe_target_groups(Names = tgNames)
        self.logger.debug(f"elbv2.describe_target_groups() response: {res}")

        if len(res['TargetGroups']) > 1:
            # *** Attention ***
            # 本番トラフィックとテストトラフィックが有効な状態（AfterAllowTestTraffic から BeforeAllowTraffic までの間）で
            # 「デプロイの停止」（即時停止）を行うとロールバックされず、2つのターゲットグループが ALB に関連付けされたままになるので、
            # どちらのターゲットグループが本番用か elbv2.describe_target_groups だけでは判断できません。
            # 結果、正しく Auto Scaling ポリシーが設定できない場合があります
            self.logger.error('Both TG-1 and TG-2 are attached alb. Can not identify it for production traffic.')

        tgArn = [tg['TargetGroupArn'] for tg in res['TargetGroups'] if len(tg['LoadBalancerArns']) > 0 and tg['LoadBalancerArns'][0] == elbArn]
        self.logger.debug(f"Target Group ARN: {tgArn}")
        return tgArn[0] if len(tgArn) > 0 else 'No TargetGroup'

    def parse_resource_label_of_target_group(self, arn):
        m = self.re_targetGroup_arn.match(arn)
        self.logger.debug(f"Resource Lable of Target Group: {m}")
        return m.group(2)

    def getResourceLabel(self):
        # https://docs.aws.amazon.com/ja_jp/autoscaling/application/APIReference/API_PredefinedMetricSpecification.html
        elbArn = self.get_load_balancer_arn(ALB_NAME)
        elbResourceLabel = self.parse_resource_label_of_elb(elbArn)

        tgArn = self.get_current_target_group_arn(tgNames = [TARGET_GROUP_NAME_1, TARGET_GROUP_NAME_2], elbArn = elbArn)
        tgResourceLabel = self.parse_resource_label_of_target_group(tgArn)

        self.logger.info(f"New Resource Lable: {elbResourceLabel}/{tgResourceLabel}")
        return f"{elbResourceLabel}/{tgResourceLabel}"

    def put_policy(self):
        try:
            resourceLabel = self.getResourceLabel()
            res = appAutoScaling.put_scaling_policy(
                PolicyName = POLICY_NAME,
                ServiceNamespace = 'ecs',
                ResourceId = f"service/{ECS_CLUSTER_NAME}/{ECS_SERVICE_NAME}",
                ScalableDimension = 'ecs:service:DesiredCount',
                PolicyType = 'TargetTrackingScaling',
                TargetTrackingScalingPolicyConfiguration = {
                    'TargetValue': REQUEST_COUNT_TARGET_VALUE,
                    'PredefinedMetricSpecification' : {
                        'PredefinedMetricType': 'ALBRequestCountPerTarget',
                        'ResourceLabel': resourceLabel
                    },
                    'ScaleOutCooldown': SCALE_OUT_COOL_DOWN,
                    'ScaleInCooldown': SCALE_IN_COOL_DOWN,
                    'DisableScaleIn': DISABLE_SCALE_IN
                }
            )
            self.logger.debug(f"appAutoScaling.put_scaling_policy() response: {res}")
        except ClientError as e:
            res = { 'error': e }
            self.logger.error(e)

        return res

def lambda_handler(event, context):

    app = AutoScalingPolicy()
    app.put_policy()

    if 'DeploymentId' in event and 'LifecycleEventHookExecutionId' in event:
        logger.info('Invoked by CodeDeploy')
        res = codedeploy.put_lifecycle_event_hook_execution_status(
            deploymentId = event['DeploymentId'],
            lifecycleEventHookExecutionId = event['LifecycleEventHookExecutionId'],
            status = 'Succeeded'
        )
        logger.debug(f"codedeploy.put_lifecycle_event_hook_execution_status() response: {res}")

    return {
        'statusCode': 200,
        'body': json.dumps(res)
    }