clickhouse-operator

Helm chart

Last changed

Request a free trial

Contact our team to test out this Helm chart and related images for free. Please also indicate any other images you would like to evaluate.

Tag:

namespaceOverride: ""

# commonLabels -- set of labels that will be applied to all the resources for the operator

commonLabels: {}

# commonAnnotations -- set of annotations that will be applied to all the resources for the operator

commonAnnotations: {}

deployment:

# look details in `kubectl explain deployment.spec.strategy`

strategy:

type: Recreate

crdHook:

# crdHook.enabled -- enable automatic CRD installation/update via pre-install/pre-upgrade hooks

# when disabled, CRDs must be installed manually using kubectl apply

enabled: true

image:

# crdHook.image.repository -- image repository for CRD installation job

repository: cgr.dev/chainguard-private/kubectl

# crdHook.image.tag -- image tag for CRD installation job

tag: latest-dev@sha256:7542d78d96ca19bd41f188066f28e95454e75f26df80d45010853d4c0d299dbc

# crdHook.image.pullPolicy -- image pull policy for CRD installation job

pullPolicy: IfNotPresent

# crdHook.imagePullSecrets -- image pull secrets for CRD installation job

# possible value format `[{"name":"your-secret-name"}]`,

# check `kubectl explain pod.spec.imagePullSecrets` for details

imagePullSecrets: []

# crdHook.resources -- resource limits and requests for CRD installation job

resources: {}

# limits:

# cpu: 100m

# memory: 128Mi

# requests:

# cpu: 100m

# memory: 128Mi

# crdHook.nodeSelector -- node selector for CRD installation job

nodeSelector: {}

# crdHook.tolerations -- tolerations for CRD installation job

tolerations: []

# crdHook.affinity -- affinity for CRD installation job

affinity: {}

# crdHook.annotations -- additional annotations for CRD installation job

annotations: {}

# crdHook.containerSecurityContext -- container security context for CRD installation job

# check `kubectl explain pod.spec.containers.securityContext` for details

containerSecurityContext: {}

# allowPrivilegeEscalation: false

# capabilities:

# drop:

# - ALL

# runAsNonRoot: true

# seccompProfile:

# type: RuntimeDefault

operator:

image:

# operator.image.registry -- optional image registry prefix (e.g. 1234567890.dkr.ecr.us-east-1.amazonaws.com)

registry: ""

# operator.image.repository -- image repository

repository: cgr.dev/chainguard-private/clickhouse-operator

# operator.image.tag -- image tag (chart's appVersion value will be used if not set)

tag: latest@sha256:db19dd2832f4537abc9722bfa2acafdefd164b027e0893e0837a5be402c65363

# operator.image.pullPolicy -- image pull policy

pullPolicy: IfNotPresent

containerSecurityContext: {}

# operator.resources -- custom resource configuration, check `kubectl explain pod.spec.containers.resources` for details

resources: {}

# limits:

# cpu: 100m

# memory: 128Mi

# requests:

# cpu: 100m

# memory: 128Mi

# operator.priorityClassName -- priority class name for the clickhouse-operator deployment, check `kubectl explain pod.spec.priorityClassName` for details

# @default -- ""

priorityClassName: ""

# operator.env -- additional environment variables for the clickhouse-operator container in deployment

# possible format value `[{"name": "SAMPLE", "value": "text"}]`

env: []

# operator.livenessProbe -- optional liveness probe for the clickhouse-operator container

# check `kubectl explain pod.spec.containers.livenessProbe` for details

# example:

# httpGet:

# path: /metrics

# port: op-metrics

# initialDelaySeconds: 10

# periodSeconds: 10

livenessProbe: null

# operator.readinessProbe -- optional readiness probe for the clickhouse-operator container

# check `kubectl explain pod.spec.containers.readinessProbe` for details

# example:

# httpGet:

# path: /metrics

# port: op-metrics

# initialDelaySeconds: 5

# periodSeconds: 5

readinessProbe: null

metrics:

enabled: true

image:

# metrics.image.registry -- optional image registry prefix (e.g. 1234567890.dkr.ecr.us-east-1.amazonaws.com)

registry: ""

100

# metrics.image.repository -- image repository

101

repository: cgr.dev/chainguard-private/clickhouse-operator-metrics-exporter

102

# metrics.image.tag -- image tag (chart's appVersion value will be used if not set)

103

tag: latest@sha256:c45d6ebb34c759bcc35f147f4bff7fce348d23555f639c2098812b8f9d1191b1

104

# metrics.image.pullPolicy -- image pull policy

105

pullPolicy: IfNotPresent

106

containerSecurityContext: {}

107

# metrics.resources -- custom resource configuration

108

resources: {}

109

# limits:

110

# cpu: 100m

111

# memory: 128Mi

112

# requests:

113

# cpu: 100m

114

# memory: 128Mi

115

116

# metrics.env -- additional environment variables for the deployment of metrics-exporter containers

117

# possible format value `[{"name": "SAMPLE", "value": "text"}]`

118

env: []

119

# metrics.livenessProbe -- optional liveness probe for the metrics-exporter container

120

# check `kubectl explain pod.spec.containers.livenessProbe` for details

121

# example:

122

# httpGet:

123

# path: /metrics

124

# port: ch-metrics

125

# initialDelaySeconds: 10

126

# periodSeconds: 10

127

livenessProbe: null

128

# metrics.readinessProbe -- optional readiness probe for the metrics-exporter container

129

# check `kubectl explain pod.spec.containers.readinessProbe` for details

130

# example:

131

# httpGet:

132

# path: /metrics

133

# port: ch-metrics

134

# initialDelaySeconds: 5

135

# periodSeconds: 5

136

readinessProbe: null

137

# imagePullSecrets -- image pull secret for private images in clickhouse-operator pod

138

# possible value format `[{"name":"your-secret-name"}]`,

139

# check `kubectl explain pod.spec.imagePullSecrets` for details

140

imagePullSecrets: []

141

# podLabels -- labels to add to the clickhouse-operator pod

142

podLabels: {}

143

# podAnnotations -- annotations to add to the clickhouse-operator pod, check `kubectl explain pod.spec.annotations` for details

144

# @default -- check the `values.yaml` file

145

podAnnotations:

146

prometheus.io/port: '8888'

147

prometheus.io/scrape: 'true'

148

clickhouse-operator-metrics/port: '9999'

149

clickhouse-operator-metrics/scrape: 'true'

150

# nameOverride -- override name of the chart

151

nameOverride: ""

152

# fullnameOverride -- full name of the chart.

153

fullnameOverride: ""

154

serviceAccount:

155

# serviceAccount.create -- specifies whether a service account should be created

156

create: true

157

# serviceAccount.annotations -- annotations to add to the service account

158

annotations: {}

159

# serviceAccount.name -- the name of the service account to use; if not set and create is true, a name is generated using the fullname template

160

name:

161

rbac:

162

# rbac.create -- specifies whether rbac resources should be created

163

create: true

164

# rbac.namespaceScoped -- specifies whether to create roles and rolebindings at the cluster level or namespace level

165

namespaceScoped: false

166

secret:

167

# secret.create -- create a secret with operator credentials

168

create: true

169

# secret.username -- operator credentials username

170

username: clickhouse_operator

171

# secret.password -- operator credentials password

172

password: clickhouse_operator_password

173

# nodeSelector -- node for scheduler pod assignment, check `kubectl explain pod.spec.nodeSelector` for details

174

nodeSelector: {}

175

# tolerations -- tolerations for scheduler pod assignment, check `kubectl explain pod.spec.tolerations` for details

176

tolerations: []

177

# affinity -- affinity for scheduler pod assignment, check `kubectl explain pod.spec.affinity` for details

178

affinity: {}

179

# podSecurityContext - operator deployment SecurityContext, check `kubectl explain pod.spec.securityContext` for details

180

podSecurityContext: {}

181

# topologySpreadConstraints - topologySpreadConstraints affinity for scheduler pod assignment, check `kubectl explain pod.spec.topologySpreadConstraints` for details

182

topologySpreadConstraints: []

183

serviceMonitor:

184

# serviceMonitor.enabled -- ServiceMonitor Custom resource is created for a [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator)

185

# In serviceMonitor will be created two endpoints ch-metrics on port 8888 and op-metrics # 9999. Ypu can specify interval, scrapeTimeout, relabelings, metricRelabelings for each endpoint below

186

enabled: false

187

# serviceMonitor.additionalLabels -- additional labels for service monitor

188

additionalLabels: {}

189

clickhouseMetrics:

190

# serviceMonitor.interval for ch-metrics endpoint --

191

interval: 30s

192

# serviceMonitor.scrapeTimeout for ch-metrics endpoint -- Prometheus ServiceMonitor scrapeTimeout. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used.

193

scrapeTimeout: ""

194

# serviceMonitor.relabelings for ch-metrics endpoint -- Prometheus [RelabelConfigs] to apply to samples before scraping

195

relabelings: []

196

# serviceMonitor.metricRelabelings for ch-metrics endpoint -- Prometheus [MetricRelabelConfigs] to apply to samples before ingestio

197

metricRelabelings: []

198

operatorMetrics:

199

# serviceMonitor.interval for op-metrics endpoint --

200

interval: 30s

201

# serviceMonitor.scrapeTimeout for op-metrics endpoint -- Prometheus ServiceMonitor scrapeTimeout. If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used.

202

scrapeTimeout: ""

203

# serviceMonitor.relabelings for op-metrics endpoint -- Prometheus [RelabelConfigs] to apply to samples before scraping

204

relabelings: []

205

# serviceMonitor.metricRelabelings for op-metrics endpoint -- Prometheus [MetricRelabelConfigs] to apply to samples before ingestio

206

metricRelabelings: []

207

# configs -- clickhouse operator configs

208

# @default -- check the `values.yaml` file for the config content (auto-generated from latest operator release)

209

configs:

210

confdFiles: null

211

configdFiles:

212

01-clickhouse-01-listen.xml: |

213

214

215

216

217

218

219

220

221

<listen_host>::</listen_host>

222

<listen_host>0.0.0.0</listen_host>

223

<listen_try>1</listen_try>

224

</yandex>

225

01-clickhouse-02-logger.xml: |

226

227

228

229

230

231

232

233

234

235

<level>debug</level>

236

<log>/var/log/clickhouse-server/clickhouse-server.log</log>

237

<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>

238

239

240

241

242

</logger>

243

</yandex>

244

01-clickhouse-03-query_log.xml: |

245

246

247

248

249

250

251

252

<query_log replace="1">

253

<database>system</database>

254

<table>query_log</table>

255

<engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>

256

<flush_interval_milliseconds>7500</flush_interval_milliseconds>

257

</query_log>

258

<query_thread_log remove="1"/>

259

</yandex>

260

01-clickhouse-04-part_log.xml: |

261

262

263

264

265

266

267

268

<part_log replace="1">

269

<database>system</database>

270

271

<engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>

272

<flush_interval_milliseconds>7500</flush_interval_milliseconds>

273

</part_log>

274

</yandex>

275

01-clickhouse-05-trace_log.xml: |-

276

277

278

279

280

281

282

283

<trace_log replace="1">

284

<database>system</database>

285

<table>trace_log</table>

286

<engine>Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day</engine>

287

<flush_interval_milliseconds>7500</flush_interval_milliseconds>

288

</trace_log>

289

</yandex>

290

files:

291

config.yaml:

292

# IMPORTANT

293

# This file is auto-generated

294

# Do not edit this file - all changes would be lost

295

# Edit appropriate template in the following folder:

296

# deploy/builder/templates-config

297

# IMPORTANT

298

299

# Template parameters available:

300

# WATCH_NAMESPACES=

301

# CH_USERNAME_PLAIN=

302

# CH_PASSWORD_PLAIN=

303

# CH_CREDENTIALS_SECRET_NAMESPACE=

304

# CH_CREDENTIALS_SECRET_NAME=clickhouse-operator

305

# VERBOSITY=1

306

307

################################################

308

309

## Watch section

310

311

################################################

312

watch:

313

# Namespaces where clickhouse-operator watches for events.

314

# Concurrently running operators should watch on different namespaces.

315

# `include` and `exclude` accept literal namespace names or regexp patterns.

316

# Empty `include` watches the operator's own namespace (or all namespaces when

317

# the operator runs in `kube-system`); use [".*"] to force watch-all elsewhere.

318

# Empty `exclude` matches none. `exclude` is applied after `include`.

319

namespaces:

320

include: []

321

322

# Behavior when ClickHouseOperatorConfiguration changes: none | restart

323

configuration:

324

onChange: restart

325

clickhouse:

326

configuration:

327

################################################

328

329

## Configuration files section

330

331

################################################

332

file:

333

# Each 'path' can be either absolute or relative.

334

# In case path is absolute - it is used as is

335

# In case path is relative - it is relative to the folder where configuration file you are reading right now is located.

336

path:

337

# Path to the folder where ClickHouse configuration files common for all instances within a CHI are located.

338

common: chi/config.d

339

# Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located.

340

host: chi/conf.d

341

# Path to the folder where ClickHouse configuration files with users' settings are located.

342

# Files are common for all instances within a CHI.

343

user: chi/users.d

344

################################################

345

346

## Configuration users section

347

348

################################################

349

user:

350

# Default settings for user accounts, created by the operator.

351

# IMPORTANT. These are not access credentials or settings for 'default' user account,

352

# it is a template for filling out missing fields for all user accounts to be created by the operator,

353

# with the following EXCEPTIONS:

354

# 1. 'default' user account DOES NOT use provided password, but uses all the rest of the fields.

355

# Password for 'default' user account has to be provided explicitly, if to be used.

356

# 2. CHOP user account DOES NOT use:

357

# - profile setting. It uses predefined profile called 'clickhouse_operator'

358

# - quota setting. It uses empty quota name.

359

# - networks IP setting. Operator specifies 'networks/ip' user setting to match operators' pod IP only.

360

# - password setting. Password for CHOP account is used from 'clickhouse.access.*' section

361

default:

362

# Default values for ClickHouse user account(s) created by the operator

363

# 1. user/profile - string

364

# 2. user/quota - string

365

# 3. user/networks/ip - multiple strings

366

# 4. user/password - string

367

# These values can be overwritten on per-user basis.

368

profile: "default"

369

quota: "default"

370

networksIP:

371

- "::1"

372

- "127.0.0.1"

373

password: "default"

374

################################################

375

376

## Configuration network section

377

378

################################################

379

network:

380

# Default host_regexp to limit network connectivity from outside

381

hostRegexpTemplate: "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"

382

################################################

383

384

## Configuration restart policy section

385

## Configuration restart policy describes what configuration changes require ClickHouse restart

386

387

################################################

388

configurationRestartPolicy:

389

rules:

390

# IMPORTANT!

391

# Special version of "*" - default version - has to satisfy all ClickHouse versions.

392

# Default version will also be used in case ClickHouse version is unknown.

393

# ClickHouse version may be unknown due to host being down - for example, because of incorrect "settings" section.

394

# ClickHouse is not willing to start in case incorrect/unknown settings are provided in config file.

395

- version: "*"

396

rules:

397

# see https://kb.altinity.com/altinity-kb-setup-and-maintenance/altinity-kb-server-config-files/#server-config-configxml-sections-which-dont-require-restart

398

# to be replaced with "select * from system.server_settings where changeable_without_restart = 'No'"

399

- settings/*: "yes"

400

# single values

401

- settings/access_control_path: "no"

402

- settings/dictionaries_config: "no"

403

- settings/max_server_memory_*: "no"

404

- settings/max_*_to_drop: "no"

405

- settings/max_concurrent_queries: "no"

406

- settings/models_config: "no"

407

- settings/user_defined_executable_functions_config: "no"

408

# structured XML

409

- settings/logger/*: "no"

410

- settings/macros/*: "no"

411

- settings/remote_servers/*: "no"

412

- settings/user_directories/*: "no"

413

# these settings should not lead to pod restarts

414

- settings/display_secrets_in_show_and_select: "no"

415

- zookeeper/*: "no"

416

- files/*.xml: "yes"

417

- files/config.d/*.xml: "yes"

418

- files/config.d/*dict*.xml: "no"

419

- files/config.d/*no_restart*: "no"

420

# exceptions in default profile

421

- profiles/default/background_*_pool_size: "yes"

422

- profiles/default/max_*_for_server: "yes"

423

- version: "21.*"

424

rules:

425

- settings/logger: "yes"

426

#################################################

427

428

## Access to ClickHouse instances

429

430

################################################

431

access:

432

# Possible values for 'scheme' are:

433

# 1. http - force http to be used to connect to ClickHouse instances

434

# 2. https - force https to be used to connect to ClickHouse instances

435

# 3. auto - either http or https is selected based on open ports

436

scheme: "auto"

437

# ClickHouse credentials (username, password and port) to be used by the operator to connect to ClickHouse instances.

438

# These credentials are used for:

439

# 1. Metrics requests

440

# 2. Schema maintenance

441

# User with these credentials can be specified in additional ClickHouse .xml config files,

442

# located in 'clickhouse.configuration.file.path.user' folder

443

username: ""

444

password: ""

445

rootCA: ""

446

# Location of the k8s Secret with username and password to be used by the operator to connect to ClickHouse instances.

447

# Can be used instead of explicitly specified username and password available in sections:

448

# - clickhouse.access.username

449

# - clickhouse.access.password

450

# Secret should have two keys:

451

# 1. username

452

# 2. password

453

secret:

454

# Empty `namespace` means that k8s secret would be looked in the same namespace where operator's pod is running.

455

namespace: ""

456

# Empty `name` means no k8s Secret would be looked for

457

458

# Port where to connect to ClickHouse instances to

459

port: 8123

460

# Timeouts used to limit connection and queries from the operator to ClickHouse instances

461

# Specified in seconds.

462

timeouts:

463

# Timout to setup connection from the operator to ClickHouse instances. In seconds.

464

connect: 5

465

# Timout to perform SQL query from the operator to ClickHouse instances. In seconds.

466

query: 4

467

################################################

468

469

## Addons specifies additional configuration sections

470

## Should it be called something like "templates"?

471

472

################################################

473

addons:

474

rules:

475

- version: "*"

476

spec:

477

configuration:

478

users:

479

profiles:

480

quotas:

481

settings:

482

files:

483

- version: ">= 23.3"

484

spec:

485

configuration:

486

###

487

### users.d is global while description depends on CH version which may vary on per-host basis

488

### In case of global-ness this may be better to implement via auto-templates

489

###

490

### As a solution, this may be applied on the whole cluster based on any of its hosts

491

###

492

### What to do when host is just created? CH version is not known prior to CH started and user config is required before CH started.

493

### We do not have any info about the cluster on initial creation

494

###

495

users:

496

"{clickhouseOperatorUser}/access_management": 1

497

"{clickhouseOperatorUser}/named_collection_control": 1

498

"{clickhouseOperatorUser}/show_named_collections": 1

499

"{clickhouseOperatorUser}/show_named_collections_secrets": 1

500

profiles:

501

quotas:

502

settings:

503

files:

504

- version: ">= 23.5"

505

spec:

506

configuration:

507

users:

508

profiles:

509

clickhouse_operator/format_display_secrets_in_show_and_select: 1

510

quotas:

511

settings:

512

513

## this may be added on per-host basis into host's conf.d folder

514

515

display_secrets_in_show_and_select: 1

516

files:

517

#################################################

518

519

## Metrics collection

520

521

################################################

522

metrics:

523

# Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances

524

# Specified in seconds.

525

timeouts:

526

# Timeout used to limit metrics collection request. In seconds.

527

# Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle.

528

# All collected metrics are returned.

529

collect: 9

530

# Regexp to match tables in system database to fetch metrics from.

531

# Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.

532

# Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.

533

tablesRegexp: "^(metrics|custom_metrics)$"

534

# List of regexps to match ClickHouse metrics to exclude from export.

535

# Regexps match internal metric names before Prometheus normalization and prefixing.

536

# Default is the per-CPU OS metrics filter shown below; set to [] to disable.

537

excludeRegexp:

538

- "^metric\\.(OS.*CPU[0-9]+|CPUFrequencyMHz_[0-9]+)$"

539

keeper:

540

configuration:

541

################################################

542

543

## Configuration files section

544

545

################################################

546

file:

547

# Each 'path' can be either absolute or relative.

548

# In case path is absolute - it is used as is

549

# In case path is relative - it is relative to the folder where configuration file you are reading right now is located.

550

path:

551

# Path to the folder where Keeper configuration files common for all instances within a CHK are located.

552

common: chk/keeper_config.d

553

# Path to the folder where Keeper configuration files unique for each instance (host) within a CHK are located.

554

host: chk/conf.d

555

# Path to the folder where Keeper configuration files with users' settings are located.

556

# Files are common for all instances within a CHI.

557

user: chk/users.d

558

################################################

559

560

## Template(s) management section

561

562

################################################

563

template:

564

chi:

565

# CHI template updates handling policy

566

# Possible policy values:

567

# - ReadOnStart. Accept CHIT updates on the operator's start only.

568

# - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply new CHITs on next regular reconcile of the CHI

569

policy: ApplyOnNextReconcile

570

# Path to the folder where ClickHouseInstallation templates .yaml manifests are located.

571

# Templates are added to the list of all templates and used when CHI is reconciled.

572

# Templates are applied in sorted alpha-numeric order.

573

path: chi/templates.d

574

chk:

575

# CHK template updates handling policy

576

# Possible policy values:

577

# - ReadOnStart. Accept CHIT updates on the operators start only.

578

# - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply new CHITs on next regular reconcile of the CHI

579

policy: ApplyOnNextReconcile

580

# Path to the folder where ClickHouseInstallation templates .yaml manifests are located.

581

# Templates are added to the list of all templates and used when CHI is reconciled.

582

# Templates are applied in sorted alpha-numeric order.

583

path: chk/templates.d

584

################################################

585

586

## Reconcile section

587

588

################################################

589

reconcile:

590

# Reconcile runtime settings

591

runtime:

592

# Max number of concurrent CHI reconciles in progress

593

reconcileCHIsThreadsNumber: 10

594

# The operator reconciles shards concurrently in each CHI with the following limitations:

595

# 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently

596

# can not be greater than 'reconcileShardsThreadsNumber'.

597

# 2. Percentage of shards being reconciled (and thus having hosts down) in each CHI concurrently

598

# can not be greater than 'reconcileShardsMaxConcurrencyPercent'.

599

# 3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward.

600

# Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage

601

602

# Max number of concurrent shard reconciles within one cluster in progress

603

reconcileShardsThreadsNumber: 5

604

# Max percentage of concurrent shard reconciles within one cluster in progress

605

reconcileShardsMaxConcurrencyPercent: 50

606

# Reconcile StatefulSet scenario

607

statefulSet:

608

# Create StatefulSet scenario

609

create:

610

# What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds

611

# Possible options:

612

# 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,

613

# do not try to fix or delete or update it, just abort reconcile cycle.

614

# Do not proceed to the next StatefulSet(s) and wait for an admin to assist.

615

# 2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards.

616

# 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.

617

onFailure: ignore

618

# Update StatefulSet scenario

619

update:

620

# How many seconds to wait for created/updated StatefulSet to be 'Ready'

621

timeout: 300

622

# How many seconds to wait between checks/polls for created/updated StatefulSet status

623

pollInterval: 5

624

# What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds

625

# Possible options:

626

# 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,

627

# do not try to fix or delete or update it, just abort reconcile cycle.

628

# Do not proceed to the next StatefulSet(s) and wait for an admin to assist.

629

# 2. rollback - delete Pod and rollback StatefulSet to previous Generation.

630

# Pod would be recreated by StatefulSet based on rollback-ed StatefulSet configuration.

631

# Follow 'abort' path afterwards.

632

# 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.

633

onFailure: abort

634

# Recreate StatefulSet scenario

635

recreate:

636

# What to do in case operator is in need to recreate StatefulSet?

637

# Possible options:

638

# 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,

639

# do not try to fix or delete or update it, just abort reconcile cycle.

640

# Do not proceed to the next StatefulSet(s) and wait for an admin to assist.

641

# 2. recreate - proceed and recreate StatefulSet.

642

643

# Triggered when PVC data loss or missing volumes are detected

644

onDataLoss: recreate

645

# Triggered when StatefulSet update fails or StatefulSet is not ready

646

onUpdateFailure: recreate

647

# Reconcile Host scenario

648

host:

649

# The operator during reconcile procedure should wait for a ClickHouse host to achieve the following conditions:

650

wait:

651

# Whether the operator during reconcile procedure should wait for a ClickHouse host:

652

# - to be excluded from a ClickHouse cluster

653

# - to complete all running queries

654

# - to be included into a ClickHouse cluster

655

# respectfully before moving forward with host reconcile

656

657

queries: true

658

include: false

659

# The operator during reconcile procedure should wait for replicas to catch-up

660

# replication delay a.k.a replication lag for the following replicas

661

replicas:

662

# All replicas (new and known earlier) are explicitly requested to wait for replication to catch-up

663

all: no

664

# New replicas only are requested to wait for replication to catch-up

665

new: yes

666

# Replication catch-up is considered to be completed as soon as replication delay

667

# a.k.a replication lag - calculated as "MAX(absolute_delay) FROM system.replicas"

668

# is within this specified delay (in seconds)

669

delay: 10

670

probes:

671

# Whether the operator during host launch procedure should wait for startup probe to succeed.

672

# In case probe is unspecified wait is assumed to be completed successfully.

673

# Default option value is to do not wait.

674

startup: no

675

# Whether the operator during host launch procedure should wait for readiness probe to succeed.

676

# In case probe is unspecified wait is assumed to be completed successfully.

677

# Default option value is to wait.

678

readiness: yes

679

# The operator during reconcile procedure should drop the following entities:

680

drop:

681

replicas:

682

# Whether the operator during reconcile procedure should drop replicas when replica is deleted

683

onDelete: yes

684

# Whether the operator during reconcile procedure should drop replicas when replica volume is lost

685

onLostVolume: yes

686

# Whether the operator during reconcile procedure should drop active replicas when replica is deleted or recreated

687

active: no

688

################################################

689

690

## Coordination with external systems during reconcile

691

692

################################################

693

coordination:

694

keeper:

695

# How long the operator waits for a referenced ClickHouseKeeper to become ready

696

# before aborting CHI reconcile. In seconds.

697

readyTimeout: 120

698

# Reaction when a referenced CHK resource changes:

699

# none — do nothing (default, backward-compatible)

700

# reconcile — trigger CHI reconcile

701

# onKeeperResourceUpdate: none

702

################################################

703

704

## Auto-recovery from aborted reconcile

705

706

################################################

707

recovery:

708

# Recovery scopes keyed by CHI state being recovered from.

709

# Each scope contains on<Event>: <action> mappings that apply while the CHI

710

# is in that state. Multi-scope design anticipates future states beyond Aborted

711

# (e.g. Failed, Broken).

712

from:

713

# Recovery from Status=Aborted

714

aborted:

715

# Action when a pod belonging to an Aborted CHI transitions to Ready:

716

# retry (default) — re-enqueue the CHI for reconcile

717

# none — do nothing, CHI stays Aborted

718

onPodReady: retry

719

# Future events (not yet implemented):

720

# onKeeperReady: retry — retry when a referenced CHK becomes ready

721

# onOperatorRestart: retry — sweep Aborted CHIs on operator startup

722

# Future scopes (not yet implemented):

723

# failed:

724

# onPodReady: retry

725

# broken:

726

# onPodReady: retry

727

# Future global policy knobs (not yet implemented) — flat peers of `from`,

728

# apply across all recovery scopes:

729

730

# Global kill-switch for auto-recovery:

731

# enabled: true

732

733

# Cap on consecutive auto-recovery attempts before giving up:

734

# retries: 5

735

736

# Minimum time between auto-recovery attempts for the same CHI:

737

# cooldown: 30s

738

739

# Exponential backoff for auto-recovery attempts:

740

# backoff:

741

# duration: 5s

742

# factor: 2

743

# maxDuration: 2m

744

################################################

745

746

## Annotations management section

747

748

################################################

749

annotation:

750

# Applied when:

751

# 1. Propagating annotations from the CHI's `metadata.annotations` to child objects' `metadata.annotations`,

752

# 2. Propagating annotations from the CHI Template's `metadata.annotations` to CHI's `metadata.annotations`,

753

# Include annotations from the following list:

754

# Applied only when not empty. Empty list means "include all, no selection"

755

include: []

756

# Exclude annotations from the following list:

757

758

################################################

759

760

## Labels management section

761

762

################################################

763

label:

764

# Applied when:

765

# 1. Propagating labels from the CHI's `metadata.labels` to child objects' `metadata.labels`,

766

# 2. Propagating labels from the CHI Template's `metadata.labels` to CHI's `metadata.labels`,

767

# Include labels from the following list:

768

# Applied only when not empty. Empty list means "include all, no selection"

769

include: []

770

# Exclude labels from the following list:

771

# Applied only when not empty. Empty list means "nothing to exclude, no selection"

772

773

# Whether to append *Scope* labels to StatefulSet and Pod.

774

# Full list of available *scope* labels check in 'labeler.go'

775

# LabelShardScopeIndex

776

# LabelReplicaScopeIndex

777

# LabelCHIScopeIndex

778

# LabelCHIScopeCycleSize

779

# LabelCHIScopeCycleIndex

780

# LabelCHIScopeCycleOffset

781

# LabelClusterScopeIndex

782

# LabelClusterScopeCycleSize

783

# LabelClusterScopeCycleIndex

784

# LabelClusterScopeCycleOffset

785

appendScope: "no"

786

################################################

787

788

## Metrics management section

789

790

################################################

791

metrics:

792

labels:

793

794

################################################

795

796

## Status management section

797

798

################################################

799

status:

800

fields:

801

action: false

802

actions: false

803

error: true

804

errors: true

805

################################################

806

807

## StatefulSet management section

808

809

################################################

810

statefulSet:

811

revisionHistoryLimit: 0

812

################################################

813

814

## Pod management section

815

816

################################################

817

pod:

818

# Grace period for Pod termination.

819

# How many seconds to wait between sending

820

# SIGTERM and SIGKILL during Pod termination process.

821

# Increase this number is case of slow shutdown.

822

terminationGracePeriod: 30

823

################################################

824

825

## Log parameters section

826

827

################################################

828

logger:

829

logtostderr: "true"

830

alsologtostderr: "false"

831

v: "1"

832

stderrthreshold: ""

833

vmodule: ""

834

log_backtrace_at: ""

835

templatesdFiles:

836

001-templates.json.example: |

837

{

838

"apiVersion": "clickhouse.altinity.com/v1",

839

"kind": "ClickHouseInstallationTemplate",

840

"metadata": {

841

"name": "01-default-volumeclaimtemplate"

842

843

"spec": {

844

"templates": {

845

"volumeClaimTemplates": [

846

{

847

"name": "chi-default-volume-claim-template",

848

"spec": {

849

"accessModes": [

850

"ReadWriteOnce"

851

852

"resources": {

853

"requests": {

854

"storage": "2Gi"

855

}

856

}

857

}

858

}

859

860

"podTemplates": [

861

{

862

"name": "chi-default-oneperhost-pod-template",

863

"distribution": "OnePerHost",

864

"spec": {

865

"containers" : [

866

{

867

"name": "clickhouse",

868

"image": "clickhouse/clickhouse-server:23.8",

869

"ports": [

870

{

871

"name": "http",

872

"containerPort": 8123

873

874

{

875

"name": "client",

876

"containerPort": 9000

877

878

{

879

"name": "interserver",

880

"containerPort": 9009

881

}

882

]

883

}

884

]

885

}

886

}

887

]

888

}

889

}

890

}

891

default-pod-template.yaml.example: |

892

apiVersion: "clickhouse.altinity.com/v1"

893

kind: "ClickHouseInstallationTemplate"

894

metadata:

895

896

spec:

897

templates:

898

podTemplates:

899

- name: default-oneperhost-pod-template

900

distribution: "OnePerHost"

901

default-storage-template.yaml.example: |

902

apiVersion: "clickhouse.altinity.com/v1"

903

kind: "ClickHouseInstallationTemplate"

904

metadata:

905

906

spec:

907

templates:

908

volumeClaimTemplates:

909

- name: default-storage-template-2Gi

910

spec:

911

accessModes:

912

- ReadWriteOnce

913

resources:

914

requests:

915

storage: 2Gi

916

readme: |-

917

Templates in this folder are packaged with an operator and available via 'useTemplate'

918

usersdFiles:

919

01-clickhouse-operator-profile.xml: |

920

921

922

923

924

925

926

<!--

927

928

# Template parameters available:

929

930

-->

931

932

933

934

<clickhouse_operator>

935

<log_queries>0</log_queries>

936

<skip_unavailable_shards>1</skip_unavailable_shards>

937

<http_connection_timeout>10</http_connection_timeout>

938

<max_concurrent_queries_for_all_users>0</max_concurrent_queries_for_all_users>

939

<os_thread_priority>0</os_thread_priority>

940

</clickhouse_operator>

941

</profiles>

942

</yandex>

943

02-clickhouse-default-profile.xml: |-

944

945

946

947

948

949

950

951

952

953

<os_thread_priority>2</os_thread_priority>

954

<log_queries>1</log_queries>

955

<connect_timeout_with_failover_ms>1000</connect_timeout_with_failover_ms>

956

<distributed_aggregation_memory_efficient>1</distributed_aggregation_memory_efficient>

957

<parallel_view_processing>1</parallel_view_processing>

958

<do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>

959

<load_balancing>nearest_hostname</load_balancing>

960

<prefer_localhost_replica>0</prefer_localhost_replica>

961

962

</default>

963

</profiles>

964

</yandex>

965

keeperConfdFiles: null

966

keeperConfigdFiles:

967

01-keeper-01-default-config.xml: |

968

969

970

971

972

973

974

975

<asynchronous_metrics_keeper_metrics_only>1</asynchronous_metrics_keeper_metrics_only>

976

<keeper_server>

977

<coordination_settings>

978

<async_replication>1</async_replication>

979

<min_session_timeout_ms>10000</min_session_timeout_ms>

980

<operation_timeout_ms>10000</operation_timeout_ms>

981

<raft_logs_level>information</raft_logs_level>

982

<session_timeout_ms>100000</session_timeout_ms>

983

<use_xid_64>1</use_xid_64>

984

</coordination_settings>

985

<hostname_checks_enabled>true</hostname_checks_enabled>

986

<log_storage_path>/var/lib/clickhouse-keeper/coordination/logs</log_storage_path>

987

<snapshot_storage_path>/var/lib/clickhouse-keeper/coordination/snapshots</snapshot_storage_path>

988

<storage_path>/var/lib/clickhouse-keeper</storage_path>

989

<tcp_port>2181</tcp_port>

990

<!--

991

Four-letter-word command allowlist.

992

993

Set explicitly to the upstream-default list so the operator-rendered

994

liveness probe (which sends `ruok` over TCP and expects `imok`) keeps

995

working even if a user adds their own keeper_server settings.

996

997

Without this, a user override that restricts the allowlist

998

(e.g. `four_letter_word_white_list: "mntr,stat"` for security)

999

would silently disable `ruok` → liveness probe always fails → CrashLoopBackOff.

1000

1001

The list mirrors ClickHouse Keeper's compiled-in default; users who want a

1002

stricter list can override this value, but they must keep `ruok` if they

1003

also use the default operator probes.

1004

-->

1005

<four_letter_word_white_list>conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro</four_letter_word_white_list>

1006

</keeper_server>

1007

<listen_host>::</listen_host>

1008

<listen_host>0.0.0.0</listen_host>

1009

<listen_try>1</listen_try>

1010

1011

1012

<level>information</level>

1013

</logger>

1014

<max_connections>4096</max_connections>

1015

</clickhouse>

1016

01-keeper-02-readiness.xml: |

1017

1018

1019

1020

1021

1022

1023

1024

<keeper_server>

1025

<http_control>

1026

1027

1028

<endpoint>/ready</endpoint>

1029

</readiness>

1030

</http_control>

1031

</keeper_server>

1032

</clickhouse>

1033

01-keeper-03-enable-reconfig.xml: |-

1034

1035

1036

1037

1038

1039

1040

1041

<keeper_server>

1042

<enable_reconfiguration>false</enable_reconfiguration>

1043

</keeper_server>

1044

</clickhouse>

1045

keeperTemplatesdFiles:

1046

readme: |-

1047

Templates in this folder are packaged with an operator and available via 'useTemplate'

1048

keeperUsersdFiles: null

1049

# additionalResources -- list of additional resources to create (processed via `tpl` function),

1050

# useful for create ClickHouse clusters together with clickhouse-operator.

1051

# check `kubectl explain chi` for details

1052

additionalResources: []

1053

# - |

1054

# apiVersion: v1

1055

# kind: ConfigMap

1056

# metadata:

1057

# name: {{ include "altinity-clickhouse-operator.fullname" . }}-cm

1058

# namespace: {{ include "altinity-clickhouse-operator.namespace" . }}

1059

# - |

1060

# apiVersion: v1

1061

# kind: Secret

1062

# metadata:

1063

# name: {{ include "altinity-clickhouse-operator.fullname" . }}-s

1064

# namespace: {{ include "altinity-clickhouse-operator.namespace" . }}

1065

# stringData:

1066

# mykey: my-value

1067

# - |

1068

# apiVersion: clickhouse.altinity.com/v1

1069

# kind: ClickHouseInstallation

1070

# metadata:

1071

# name: {{ include "altinity-clickhouse-operator.fullname" . }}-chi

1072

# namespace: {{ include "altinity-clickhouse-operator.namespace" . }}

1073

# spec:

1074

# configuration:

1075

# clusters:

1076

# - name: default

1077

# layout:

1078

# shardsCount: 1

1079

1080

dashboards:

1081

# dashboards.enabled -- provision grafana dashboards as configMaps (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-8.3.4/charts/grafana/values.yaml#L778 )

1082

enabled: false

1083

# dashboards.additionalLabels -- labels to add to a secret with dashboards

1084

additionalLabels:

1085

# dashboards.additionalLabels.grafana_dashboard - will watch when official grafana helm chart sidecar.dashboards.enabled=true

1086

grafana_dashboard: ""

1087

# dashboards.annotations -- annotations to add to a secret with dashboards

1088

annotations:

1089

# dashboards.annotations.grafana_folder -- folder where will place dashboards, requires define values in official grafana helm chart sidecar.dashboards.folderAnnotation: grafana_folder

1090

grafana_folder: clickhouse-operator

1091

The trusted source for open source

Talk to an expert

Privacy

Terms

© 2026 Chainguard, Inc. All Rights Reserved.
Chainguard® and the Chainguard logo are registered trademarks of Chainguard, Inc. in the United States and/or other countries.
The other respective trademarks mentioned on this page are owned by the respective companies and use of them does not imply any affiliation or endorsement.

clickhouse-operator

The trusted source for open source

Product

Solutions

Customers

Resources

Company