What
Let's say, for the sake of this short note, that you have:
- Cloudflare DNS which you use to expose your service to the world (or to your internal tailscale/VPN/home network)
- Caddy is your reverse proxy of choice
- Nomad is your deployment system of choice
Well, now, here is how I synchronize DNS records on Cloudflare based on the service discovery in Nomad / Consul using some shortcuts and a very small Python script...
Result of this process is that if I have a service with name "chronograf" my DNS record chronograf.mycooldomain.com gets updated.
How
First, the nomad job definition:
job "internal-proxy" {
datacenters = ["DC1"]
type = "service"
constraint {
attribute = "${attr.unique.hostname}"
value = "pluto"
}
group "main" {
ephemeral_disk {
migrate = true
size = 150
sticky = true
}
task "caddy" {
}
task "syncer" {
driver = "docker"
config {
image = "python:3.10.4-slim-bullseye"
volumes = [
"local/:/etc/dns-sync"
]
command = "python3"
args = ["/etc/dns-sync/syncer.py"]
}
env {
ZONE_ID_FILE = "/etc/dns-sync/cf_zone_id"
CF_API_TOKEN_FILE = "/etc/dns-sync/cf_api_key"
DNS_MAPPING_FILE = "/etc/dns-sync/records.txt"
}
template {
data = <<EOF
[[ fileContents "syncer.py" ]]
EOF
destination = "local/syncer.py"
}
template {
data = "{{ key \"cloudFlare/zoneId\" }}"
destination = "local/cf_zone_id"
change_mode = "signal"
change_signal = "SIGUSR1"
}
template {
data = "{{ key \"cloudFlare/cfApi\" }}"
destination = "local/cf_api_key"
change_mode = "signal"
change_signal = "SIGUSR1"
}
template {
data = <<EOF
{{range $tag, $services := services | byTag}}{{ if eq $tag "expose-internal" }}{{range $services}}{{ .Name }}.{{ key "cloudFlare/domain" }}|{{ with node "pluto" }}{{ .Node.Address }}{{ end }}
{{end}}{{end}}{{end}}
EOF
destination = "local/records.txt"
change_mode = "signal"
change_signal = "SIGUSR1"
}
resources {
cpu = 100
memory = 100
}
}
}
}
Some assumptions I took in the job above:
- I am using Consul as key/value store and utilizing it to get the services I am interested in and I am using Consul Templating to get this going
- I only want to expose services that are tagged with
expose-internal
- operationally, I "know" that this will be deployed only on a node with name
pluto,
- my personal Cloudflare API token and the zone ID which I plan on syncing are stored in Consul K/V storage
- also, my domain under which I am exposing services is also in Consul K/V storage
- I construct files which are updated automatically by Consul when/if K/V settings change OR if a new service mapping becomes available
Now, the script that does the syncing Nomad -> Cloudflare using pure Python 3.
import argparse
import json
import logging
import os
import signal
import sys
import urllib.parse
import urllib.request
from typing import Dict, List, Optional
def read_secret_multi(secret_name_file: str) -> List[str]:
with open(read_env_or_fail(secret_name_file), 'r') as env_file_file:
return [x.strip() for x in env_file_file.readlines()]
def read_secret(secret_name_file: str) -> str:
with open(read_env_or_fail(secret_name_file), 'r') as env_file_file:
return env_file_file.readline()
def read_env_or_fail(secret_name_file):
env_file = os.getenv(secret_name_file, None)
if env_file is None:
logging.error(f"Environment variable {secret_name_file} was not present, giving up")
sys.exit(1)
return env_file
class Syncer:
def __init__(self, zone_id: str, cf_api_token: str, dns_records: List[str]):
self.zone_id = zone_id
self.cf_api_token = cf_api_token
self.mappings = {x.split('|')[0]: x.split('|')[1] for x in dns_records if x}
def cf_api(self, path: str, method: Optional[str] = 'GET', data: Optional[Dict] = None):
req = urllib.request.Request(
url=f'https://api.cloudflare.com/client/v4/{path}',
headers={
"Authorization": f"Bearer {self.cf_api_token}",
"Content-Type": "application/json",
},
data=json.dumps(data).encode('utf-8') if data else None,
method=method,
)
with urllib.request.urlopen(req) as f:
return json.loads(f.read().decode('utf-8'))
def sync(self):
logging.info("Syncing records...")
records = self.cf_api(f'zones/{self.zone_id}/dns_records')['result']
cf_records = {rec['name']: rec for rec in records}
for our_mapping, target in self.mappings.items():
cf_mapping: Dict = cf_records.get(our_mapping, None)
if cf_mapping:
if cf_mapping['content'] == target and cf_mapping['type'] == 'A':
logging.info(f"Identical mapping on CF for: {our_mapping}, ignoring")
else:
logging.info(f"Updating existing mapping on CF for: {our_mapping} -> {target}")
self.cf_api(path=f"zones/{self.zone_id}/dns_records/{cf_mapping['id']}", method='PATCH',
data=self.make_cf_record_dto(our_mapping, target))
else:
logging.info(f"Adding new mapping on CF for: {our_mapping} -> {target}")
self.cf_api(path=f'zones/{self.zone_id}/dns_records', method='POST',
data=self.make_cf_record_dto(our_mapping, target))
@staticmethod
def make_cf_record_dto(our_mapping, target):
return {
"type": "A",
"name": our_mapping,
"content": target,
"proxied": False
}
def run():
Syncer(
zone_id=read_secret("ZONE_ID_FILE"),
cf_api_token=read_secret("CF_API_TOKEN_FILE"),
dns_records=read_secret_multi("DNS_MAPPING_FILE"),
).sync()
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='syncer.py', description="Automation script for Cloudflare record syncing")
parser.add_argument('--debug', default=False, required=False, action='store_true', dest="debug",
help='debug flag')
args = parser.parse_args()
if args.debug:
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.getLogger().setLevel(logging.INFO)
run()
signal.signal(signal.SIGUSR1, lambda sig, frame: run())
signal.signal(signal.SIGINT, lambda sig, frame: sys.exit(0))
signal.signal(signal.SIGTERM, lambda sig, frame: sys.exit(0))
while True:
logging.info('Waiting for signals...')
signal.pause()
