From 533f678ab78d6beab1eb358402bd50c2c83f065f Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 10 Mar 2026 17:22:58 +0300 Subject: [PATCH] =?UTF-8?q?=D0=90=D0=B2=D1=82=D0=BE=D0=BC=D0=B0=D1=82?= =?UTF-8?q?=D0=B8=D0=B7=D0=B0=D1=86=D0=B8=D0=B8=20=D1=80=D0=B0=D0=B7=D0=B2?= =?UTF-8?q?=D1=91=D1=80=D1=82=D1=8B=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F=20Kubern?= =?UTF-8?q?etes-=D0=BA=D0=BB=D0=B0=D1=81=D1=82=D0=B5=D1=80=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + .terraform.lock.hcl | 16 +++ .terraform/modules/modules.json | 1 + .../bpg/proxmox/0.98.1/linux_amd64 | 1 + .../bpg/proxmox/0.98.1/linux_amd64.lock | 0 .../hashicorp/local/2.7.0/linux_amd64 | 1 + .../hashicorp/local/2.7.0/linux_amd64.lock | 0 README.md | 51 +++++++++ locals.tf | 43 ++++++++ main.tf | 27 +++++ modules/k8s-node/cloud-config/node-base.yml | 20 ++++ modules/k8s-node/locals.tf | 58 ++++++++++ modules/k8s-node/main.tf | 71 ++++++++++++ modules/k8s-node/outputs.tf | 17 +++ modules/k8s-node/variables.tf | 66 +++++++++++ outputs.tf | 11 ++ providers.tf | 20 ++++ terraform.tfvars | 24 ++++ variables.tf | 103 ++++++++++++++++++ 19 files changed, 532 insertions(+) create mode 100644 .gitignore create mode 100644 .terraform.lock.hcl create mode 100644 .terraform/modules/modules.json create mode 120000 .terraform/providers/registry.opentofu.org/bpg/proxmox/0.98.1/linux_amd64 create mode 100644 .terraform/providers/registry.opentofu.org/bpg/proxmox/0.98.1/linux_amd64.lock create mode 120000 .terraform/providers/registry.opentofu.org/hashicorp/local/2.7.0/linux_amd64 create mode 100644 .terraform/providers/registry.opentofu.org/hashicorp/local/2.7.0/linux_amd64.lock create mode 100644 README.md create mode 100644 locals.tf create mode 100644 main.tf create mode 100644 modules/k8s-node/cloud-config/node-base.yml create mode 100644 modules/k8s-node/locals.tf create mode 100644 modules/k8s-node/main.tf create mode 100644 modules/k8s-node/outputs.tf create mode 100644 modules/k8s-node/variables.tf create mode 100644 outputs.tf create mode 100644 providers.tf create mode 100644 terraform.tfvars create mode 100644 variables.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b8aa1b2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +ssh + diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl new file mode 100644 index 0000000..67d093d --- /dev/null +++ b/.terraform.lock.hcl @@ -0,0 +1,16 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/bpg/proxmox" { + version = "0.98.1" + hashes = [ + "h1:/3n9NevrIwRAI/0HOEbeO4uQfCCivjspG8pDgzhqOSU=", + ] +} + +provider "registry.opentofu.org/hashicorp/local" { + version = "2.7.0" + hashes = [ + "h1:uWL9nhlxLY2xW3GMh8IFQv0S1UP3HMlN/B+2Nr7fsZE=", + ] +} diff --git a/.terraform/modules/modules.json b/.terraform/modules/modules.json new file mode 100644 index 0000000..23bc13d --- /dev/null +++ b/.terraform/modules/modules.json @@ -0,0 +1 @@ +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"cluster","Source":"./modules/k8s-node","Dir":"modules/k8s-node"}]} \ No newline at end of file diff --git a/.terraform/providers/registry.opentofu.org/bpg/proxmox/0.98.1/linux_amd64 b/.terraform/providers/registry.opentofu.org/bpg/proxmox/0.98.1/linux_amd64 new file mode 120000 index 0000000..e31b2bd --- /dev/null +++ b/.terraform/providers/registry.opentofu.org/bpg/proxmox/0.98.1/linux_amd64 @@ -0,0 +1 @@ +/home/andy/.terraform.d/plugins/registry.opentofu.org/bpg/proxmox/0.98.1/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.opentofu.org/bpg/proxmox/0.98.1/linux_amd64.lock b/.terraform/providers/registry.opentofu.org/bpg/proxmox/0.98.1/linux_amd64.lock new file mode 100644 index 0000000..e69de29 diff --git a/.terraform/providers/registry.opentofu.org/hashicorp/local/2.7.0/linux_amd64 b/.terraform/providers/registry.opentofu.org/hashicorp/local/2.7.0/linux_amd64 new file mode 120000 index 0000000..9a09af6 --- /dev/null +++ b/.terraform/providers/registry.opentofu.org/hashicorp/local/2.7.0/linux_amd64 @@ -0,0 +1 @@ +/home/andy/.terraform.d/plugins/registry.opentofu.org/hashicorp/local/2.7.0/linux_amd64 \ No newline at end of file diff --git a/.terraform/providers/registry.opentofu.org/hashicorp/local/2.7.0/linux_amd64.lock b/.terraform/providers/registry.opentofu.org/hashicorp/local/2.7.0/linux_amd64.lock new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md new file mode 100644 index 0000000..2253419 --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +Что содержит проект + +Модуль OpenTofu для создания Kubernetes-нод в Proxmox. +Генерацию cloud-init конфигураций. +Параметризованную выдачу VMID, IP, hostname. +Готовый root-конфиг, использующий модуль k8s-node. +Возможность добавлять и удалять конкретные ноды без count. +Подготовку виртуальных машин к установке Kubernetes. + +Структура репозитория +├── README.md +├── locals.tf +├── main.tf +├── modules +│ └── k8s-node +│ ├── cloud-config +│ ├── locals.tf +│ ├── main.tf +│ ├── outputs.tf +│ └── variables.tf +├── outputs.tf +├── providers.tf +├── terraform.tfstate +├── terraform.tfstate.backup +├── terraform.tfvars +└── variables.tf + +Быстрый старт +1. Инициализация проекта +tofu init +2. Проверка плана +tofu plan +3. Создание инфраструктуры +tofu apply + +После выполнения этой команды Proxmox создаст виртуальные машины, сгенерирует userdata для cloud-init и развернёт требуемые ноды. + +Управление нодами +Модуль принимает объект вида: + +nodes = { + master1 = { role = "master", cpu = 2, memory = 4096 } + worker1 = { role = "worker", cpu = 2, memory = 4096 } + worker2 = { role = "worker", cpu = 2, memory = 4096 } +} + +Вы можете: + +Добавить новую ноду, просто вписав её в map. +Удалить ноду, удалив её ключ из map. +Иметь несколько кластеров, копируя модуль в разные окружения. diff --git a/locals.tf b/locals.tf new file mode 100644 index 0000000..23ea6e0 --- /dev/null +++ b/locals.tf @@ -0,0 +1,43 @@ +locals { + ssh_public_key = trimspace(file("./ssh/id_terraform.pub")) + + nodes = { + master1 = { + role = "master" + cpu = var.master_cpu + memory = var.master_memory + disk = var.master_disk + datastore = var.master_datastore + ip_offset = var.master_ip_offset + } + + worker1 = { + role = "worker" + cpu = var.worker_cpu + memory = var.worker_memory + disk = var.worker_disk + datastore = var.worker_datastore + ip_offset = var.worker_ip_offset + } + +# worker2 = { +# role = "worker" +# cpu = var.worker_cpu +# memory = var.worker_memory +# disk = var.worker_disk +# datastore = var.worker_datastore +# ip_offset = var.worker_ip_offset +# } + + worker3 = { + role = "worker" + cpu = var.worker_cpu + memory = var.worker_memory + disk = var.worker_disk + datastore = var.worker_datastore + ip_offset = var.worker_ip_offset + } + } + + +} diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..6428e01 --- /dev/null +++ b/main.tf @@ -0,0 +1,27 @@ +data "local_file" "ssh_key" { + filename = pathexpand("./ssh/id_terraform.pub") +} + +module "cluster" { + source = "./modules/k8s-node" + + nodes = local.nodes + ssh_key = trimspace(data.local_file.ssh_key.content) + + hostname_prefix = var.hostname_prefix + cluster_ip_start = var.cluster_ip_start + master_vmid_start = var.master_vmid_start + worker_vmid_start = var.worker_vmid_start + + cloudinit_datastore = var.cloudinit_datastore + proxmox_node = var.proxmox_node + + node_bridge = var.node_bridge + image_datastore = var.image_datastore + image_file = var.image_file + disk_interface = var.disk_interface + + network_base = var.network_base + network_cidr = var.network_cidr + cluster_gateway = var.cluster_gateway +} diff --git a/modules/k8s-node/cloud-config/node-base.yml b/modules/k8s-node/cloud-config/node-base.yml new file mode 100644 index 0000000..4579ba9 --- /dev/null +++ b/modules/k8s-node/cloud-config/node-base.yml @@ -0,0 +1,20 @@ +#cloud-config + +timezone: Europe/Moscow + +users: + - default + - name: ubuntu + groups: [sudo] + shell: /bin/bash + ssh_authorized_keys: + - ${ssh_key} + +package_update: true + +packages: + - qemu-guest-agent + +runcmd: + - systemctl enable --now qemu-guest-agent + - hostnamectl set-hostname ${hostname} diff --git a/modules/k8s-node/locals.tf b/modules/k8s-node/locals.tf new file mode 100644 index 0000000..344f7fc --- /dev/null +++ b/modules/k8s-node/locals.tf @@ -0,0 +1,58 @@ +locals { + # ssh-ключ приходит снаружи, файл не читаем + ssh_public_key = var.ssh_key + + # Разделяем ноды по ролям + masters = { + for name, node in var.nodes : + name => node if node.role == "master" + } + + workers = { + for name, node in var.nodes : + name => node if node.role == "worker" + } + + # Даём каждой ноде индекс внутри своей роли (master1, master2, worker1...) + # Индекс определяется по отсортированным именам, чтобы был стабильным. + indexed_masters = { + for name, node in local.masters : + name => merge(node, { + index = index(sort(keys(local.masters)), name) + 1 + }) + } + + indexed_workers = { + for name, node in local.workers : + name => merge(node, { + index = index(sort(keys(local.workers)), name) + 1 + }) + } + + # Общая карта нод + nodes = merge(local.indexed_masters, local.indexed_workers) + + # IP-адреса + ip_map = { + for name, node in local.nodes : + name => var.cluster_ip_start + node.ip_offset + node.index + } + + # VMID: разные диапазоны для master/worker + vmid_map = { + for name, node in local.nodes : + name => ( + node.role == "master" + ? var.master_vmid_start + node.index + : var.worker_vmid_start + node.index + ) + } + + # hostname: prefix-role-index (k8s-master-1, k8s-worker-2 и т.п.) + hostname_map = { + for name, node in local.nodes : + name => "${var.hostname_prefix}-${node.role}-${node.index}" + } +} + + diff --git a/modules/k8s-node/main.tf b/modules/k8s-node/main.tf new file mode 100644 index 0000000..68ad267 --- /dev/null +++ b/modules/k8s-node/main.tf @@ -0,0 +1,71 @@ +terraform { + required_providers { + proxmox = { + source = "registry.opentofu.org/bpg/proxmox" + } + } +} +#Создание cloud-init для каждой ноды + +resource "proxmox_virtual_environment_file" "cloudinit" { + for_each = local.nodes + content_type = "snippets" + datastore_id = var.cloudinit_datastore + node_name = var.proxmox_node + + source_raw { + file_name = "${each.key}.yml" + + data = templatefile( + "${path.module}/cloud-config/node-base.yml", + { + hostname = local.hostname_map[each.key] + ssh_key = local.ssh_public_key + } + ) + } +} +# Создание виртуальных машин + +resource "proxmox_virtual_environment_vm" "nodes" { + for_each = local.nodes + + name = local.hostname_map[each.key] + node_name = var.proxmox_node + vm_id = local.vmid_map[each.key] + + agent { + enabled = true + } + + cpu { + cores = each.value.cpu + } + + memory { + dedicated = each.value.memory + } + + network_device { + bridge = var.node_bridge + } + + disk { + datastore_id = each.value.datastore + import_from = "${var.image_datastore}:${var.image_file}" + interface = var.disk_interface + size = each.value.disk + } + + initialization { + datastore_id = each.value.datastore + user_data_file_id = proxmox_virtual_environment_file.cloudinit[each.key].id + + ip_config { + ipv4 { + address = "${var.network_base}.${local.ip_map[each.key]}/${var.network_cidr}" + gateway = var.cluster_gateway + } + } + } +} diff --git a/modules/k8s-node/outputs.tf b/modules/k8s-node/outputs.tf new file mode 100644 index 0000000..cc91435 --- /dev/null +++ b/modules/k8s-node/outputs.tf @@ -0,0 +1,17 @@ +output "ip_addresses" { + description = "IP addresses of all created nodes" + value = { + for name, _ in local.nodes : + name => proxmox_virtual_environment_vm.nodes[name].ipv4_addresses[1][0] + } +} + +output "hostnames" { + description = "Hostnames of all created nodes" + value = local.hostname_map +} + +output "vmids" { + description = "VMIDs of all created nodes" + value = local.vmid_map +} diff --git a/modules/k8s-node/variables.tf b/modules/k8s-node/variables.tf new file mode 100644 index 0000000..4722676 --- /dev/null +++ b/modules/k8s-node/variables.tf @@ -0,0 +1,66 @@ +variable "ssh_key" { + type = string +} + +variable "nodes" { + type = map(object({ + role = string + cpu = number + memory = number + disk = number + datastore = string + ip_offset = number + })) +} + +variable "hostname_prefix" { + type = string +} + +variable "cluster_ip_start" { + type = number +} + +variable "master_vmid_start" { + type = number +} + +variable "worker_vmid_start" { + type = number +} + +variable "cloudinit_datastore" { + type = string +} + +variable "proxmox_node" { + type = string +} + +variable "node_bridge" { + type = string +} + +variable "image_datastore" { + type = string +} + +variable "image_file" { + type = string +} + +variable "disk_interface" { + type = string +} + +variable "network_base" { + type = string +} + +variable "network_cidr" { + type = number +} + +variable "cluster_gateway" { + type = string +} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..3721c93 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,11 @@ +output "nodes_ipv4" { + value = module.cluster.ip_addresses +} + +output "nodes_hostnames" { + value = module.cluster.hostnames +} + +output "nodes_vmid" { + value = module.cluster.vmids +} diff --git a/providers.tf b/providers.tf new file mode 100644 index 0000000..e1630d9 --- /dev/null +++ b/providers.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + proxmox = { + source = "registry.opentofu.org/bpg/proxmox" + } + } +} + +provider "proxmox" { + endpoint = var.proxmox_endpoint + api_token = "${var.proxmox_token_id}=${var.proxmox_token_secret}" + insecure = true + + ssh { + agent = true + username = "root" + private_key = file("./ssh/id_terraform") + } + +} diff --git a/terraform.tfvars b/terraform.tfvars new file mode 100644 index 0000000..aeecaae --- /dev/null +++ b/terraform.tfvars @@ -0,0 +1,24 @@ +proxmox_endpoint = "https://185.78.29.7:8006/api2/json" +proxmox_token_id = "terraform@pve!tf" +proxmox_token_secret = "API_TOKEN" + +master_vmid_start = 4000 +worker_vmid_start = 4010 + + +master_cpu = 2 +master_memory = 2048 +master_disk = 20 +master_datastore = "local" + +worker_cpu = 2 +worker_memory = 4096 +worker_disk = 30 +worker_datastore = "local" + +image_datastore = "local" +image_file = "import/ubuntu-24.qcow2" +cluster_gateway = "10.10.10.1" +network_base = "10.10.10" +network_cidr = "24" +cluster_ip_start = 40 diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..e407d56 --- /dev/null +++ b/variables.tf @@ -0,0 +1,103 @@ +variable "proxmox_endpoint" {} +variable "proxmox_token_id" {} +variable "proxmox_token_secret" {} + +variable "proxmox_node" { + type = string + default = "px" +} + +variable "cloudinit_datastore" { + type = string + default = "local" +} + +variable "disk_interface" { + type = string + default = "virtio0" +} + +variable "image_datastore" { + type = string + default = "local" +} + +variable "image_file" { + type = string + default = "import/ubuntu-24.qcow2" +} + +variable "hostname_prefix" { + type = string + default = "k8s" +} + +variable "master_cpu" { + default = 2 +} + +variable "worker_cpu" { + default = 2 +} + +variable "master_memory" { + default = 4096 +} + +variable "worker_memory" { + default = 4096 +} + +variable "master_disk" { + default = 20 +} + +variable "worker_disk" { + default = 20 +} + +variable "network_base" { + default = "10.10.10" +} + +variable "network_cidr" { + default = "24" +} + +variable "cluster_gateway" { + default = "10.10.10.1" +} + +variable "cluster_ip_start" { + default = 40 +} + +variable "master_ip_offset" { + default = 0 +} + +variable "worker_ip_offset" { + default = 5 +} + +variable "node_bridge" { + default = "vmbr1" +} +variable "master_datastore" { + type = string + default = "local" +} + +variable "worker_datastore" { + type = string + default = "local" +} +variable "master_vmid_start" { + type = number + default = 2000 +} + +variable "worker_vmid_start" { + type = number + default = 2010 +}