TerraformでGKEクラスタとBigQueryを立てる

2018-05-29 gcp kubernetes terraform

GKEクラスタからBigQueryを読み書きすることを想定している。

TerraformでVPCを管理するmoduleを作る - sambaiz-net

Kubernetesの1PodでAppとfluentdコンテナを動かしてBigQueryに送る - sambaiz-net

GKE

google_container_cluster

oauth_scopeにbigqueryを付けている。

resource "google_container_cluster" "sample" {
  name               = "${var.cluster_name}"
  description        = "sample k8s cluster"
  zone               = "${var.gcp_zone}"
  initial_node_count = "${var.initial_node_count}"

  master_auth {
    username = "${var.master_username}"
    password = "${var.master_password}"
  }

  node_config {
    machine_type = "${var.node_machine_type}"
    disk_size_gb = "${var.node_disk_size}"

    oauth_scopes = [
      "https://www.googleapis.com/auth/compute",
      "https://www.googleapis.com/auth/devstorage.read_only",
      "https://www.googleapis.com/auth/logging.write",
      "https://www.googleapis.com/auth/monitoring",
      "https://www.googleapis.com/auth/bigquery",
    ]
  }
}

variable "env" {
  description = "system env"
}
variable "gcp_zone" {
  description = "GCP zone, e.g. us-east1-b"
  default = "us-east1-b"
}

variable "cluster_name" {
  description = "Name of the K8s cluster"
}

variable "initial_node_count" {
  description = "Number of worker VMs to initially create"
  default = 1
}

variable "master_username" {
  description = "Username for accessing the Kubernetes master endpoint"
}

variable "master_password" {
  description = "Password for accessing the Kubernetes master endpoint"
}

variable "node_machine_type" {
  description = "GCE machine type"
  default = "n1-standard-1"
}

variable "node_disk_size" {
  description = "Node disk size in GB"
  default = "20"
}

BigQuery

resource "google_bigquery_dataset" "sample" {
  dataset_id  = "${var.dataset_id}"
  description = "sample dataset"
  location    = "${var.dataset_location}"

  labels {
    env = "${var.env}"
  }
}

resource "google_bigquery_table" "sample" {
  dataset_id = "${google_bigquery_dataset.sample.dataset_id}"
  table_id   = "sample"
  schema     = "${file("bigquery/sample/schema.json")}"

  time_partitioning {
    type  = "DAY"
  }

  labels {
    env = "${var.env}"
  }
}

variable "env" {
  description = "system env"
}

variable "dataset_id" {
  description = "dataset ID"
}

variable "dataset_location" {
  description = "dataset location one of [US EU]"
  default     = "US"
}

schema.jsonはこんな感じ。

[
    {
        "name": "foo",
        "type": "FLOAT64",
        "mode": "NULLABLE",
        "description": "foo"
    }
]

呼び出し元

IAMからサービスアカウントを作成し、credentialをダウンロードする。 backendはgcs。gcloud auth application-default login で認証しておく。

provider "google" {
  credentials = "${file("sample-credentials.json")}"
  project     = "sample"
}

terraform {
  backend "gcs" {
    bucket = "sample-tfstate"
    prefix = "prd"
  }
}

module "sample-cluster" {
  source             = "./gke"
  gcp_zone           = "us-west1-a"
  env                = "prd"
  cluster_name       = "sample"
  initial_node_count = 1
  master_username    = "master"
  master_password    = "ae9fqAwfGefeweV"
  node_machine_type  = "n1-standard-2"
  node_disk_size     = "20"
}

module "sample-bigquery" {
  source           = "./bigquery/sample"
  env              = "prd"
  dataset_id       = "sample"
  dataset_location = "US"
}