EKS cluster autoscaler

1. Enable CA in eks-worker-nodes.tf

# Using the new feature from reinvent:19 to provisioning node automatically without the need 
# for EC2 provisioning.  EKS-optimized AMIs will be used automatically for each node. 
# Nodes launched as part of a managed node group are automatically tagged for auto-discovery 
# by k8s cluster autoscaler. 
# https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html 
# https://www.terraform.io/docs/providers/aws/r/eks_node_group.html 
resource "aws_eks_node_group" "co-ec-eks-node-group" {
 cluster_name    = aws_eks_cluster.co-ec-eks-cluster.name    
 node_group_name = "co-ec-eks-node-group-${local.vpc_id}"   
 node_role_arn   = aws_iam_role.co-ec-eks-node-iam-role.arn   
 subnet_ids      = local.subnet_ids  instance_types  = [var.instance_type]
  scaling_config {
    desired_size = 3     
    max_size     = 8    
    min_size     = 3  }
  depends_on = [
    aws_iam_role_policy_attachment.co-ec-eks-node-AmazonEKSWorkerNodePolicy,
    aws_iam_role_policy_attachment.co-ec-eks-node-AmazonEKS_CNI_Policy,
    aws_iam_role_policy_attachment.co-ec-eks-node-AmazonEC2ContainerRegistryReadOnly,
    aws_iam_role_policy_attachment.co-ec-eks-node-CloudWatchAgentServerPolicy,
    aws_iam_role_policy_attachment.co-ec-eks-node-AmazonDynamoDBFullAccess,
  ]
}

2. Add new IAM policy to EKS node IAM role

resource "aws_iam_role_policy" "co-ec-eks-node-auto-scale-policy" {
  name = "co-ec-eks-node-auto-scale-policy"   
  role = aws_iam_role.co-ec-eks-node-iam-role.id
  policy = <<-EOF
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "autoscaling:DescribeAutoScalingGroups",
                "autoscaling:DescribeAutoScalingInstances",
                "autoscaling:DescribeLaunchConfigurations",
                "autoscaling:DescribeTags",
                "autoscaling:SetDesiredCapacity",
                "autoscaling:TerminateInstanceInAutoScalingGroup"
            ],
            "Resource": "*"
        }
    ]
}

(you can also add such policy from AWS Console to the Node IAM Role)

3. Download the sample file and rename it to cluster_autoscaler_asg, yaml file:

https://aws.amazon.com/premiumsupport/knowledge-center/eks-cluster-autoscaler-setup/

4. Make following changes to the yaml file:

<             - --expander=least-waste
<             - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/<YOUR CLUSTER NAME>
---
>             - --nodes={{MIN_NODE}}:{{MAX_NODE}}:{{K8S_NODE_ASG}}

One example is:

    - --nodes=1:10:eks-22b82878-2fa8-201b-8842-8f7f1aeeeee

5. Apply autoscaler deployment:

cat cluster-autoscaler-asg.yaml | sed "s/{{K8S_NODE_ASG}}/$K8S_NODE_ASG/;s/{{MIN_NODE}}/$MIN_NODE/;s/{{MAX_NODE}}/$MAX_NODE/" | kubectl apply -f -

References

https://aws.amazon.com/premiumsupport/knowledge-center/eks-cluster-autoscaler-setup/

https://dev.to/wingkwong/autoscaling-an-eks-cluster-with-cluster-autoscaler-30e9 (for instruction)