document and keep processes for host and lxc standardization

portainer/docker-compose.yml

@@ -1,12 +0,0 @@
-services:
-  portainer:
-    image: portainer/portainer-ce:latest
-    container_name: portainer
-    restart: always
-    ports:
-      - "8000:8000"   # Optional: for edge agent (can be removed if not used)
-      - "9443:9443"   # HTTPS UI (recommended)
-      - "9000:9000"   # HTTP UI (deprecated, optional)
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - /docker/portainer/data:/data

~host-setups/LXC-bootstrap.sh

@@ -0,0 +1,69 @@
+#update core packages
+apt update && apt upgrade -y
+
+#apply all the packages I need
+apt install -y ca-certificates curl ethtool iptables-persistent git htop
+
+#enable IP forwarding
+Enable IP Forwarding (Persistent)
+sed -i 's/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/' /etc/sysctl.conf
+grep -qF "net.ipv4.ip_forward=1" /etc/sysctl.conf || echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf
+sysctl -p
+
+#add the NAT rule for IPtables and save
+iptables -C POSTROUTING -t nat -s 172.16.0.0/12 -d 172.16.201.0/24 -j MASQUERADE 2>/dev/null || \
+iptables -t nat -I POSTROUTING 1 -s 172.16.0.0/12 -d 172.16.201.0/24 -j MASQUERADE
+netfilter-persistent save
+
+#change the virtio settings and Apply checksum fix immediately
+if ! grep -q "post-up /sbin/ethtool" /etc/network/interfaces; then
+    sed -i '/gateway/a \        post-up /sbin/ethtool -K eth0 tx off rx off' /etc/network/interfaces
+fi
+ethtool -K eth0 tx off rx off
+
+#Enable Root SSH Login
+sed -i 's/^#*PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+grep -q "^PermitRootLogin yes" /etc/ssh/sshd_config || echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+systemctl restart ssh
+
+#install the keyring for the docker repo
+install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
+chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+tee /etc/apt/sources.list.d/docker.sources <<EOF
+Types: deb
+URIs: https://download.docker.com/linux/debian
+Suites: $(. /etc/os-release && echo "$VERSION_CODENAME")
+Components: stable
+Signed-By: /etc/apt/keyrings/docker.asc
+EOF
+
+#update from the docker repo and install docker
+apt update && apt install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+
+#pre-empt that pesky json is a folder issue
+rm -rf /root/.docker/config.json
+mkdir -p /root/.docker
+echo "{}" > /root/.docker/config.json 
+
+#sanitize the LXC  
+systemctl stop docker
+rm -f /etc/docker/key.json
+rm -f /etc/ssh/ssh_host_*
+truncate -s 0 /etc/machine-id
+rm -f /var/lib/dbus/machine-id
+ln -s /etc/machine-id /var/lib/dbus/machine-id
+
+#Clean Apt Cache & Logs
+apt clean
+rm -rf /var/lib/apt/lists/*
+rm -rf /var/log/*.log
+rm -rf /var/log/journal/*
+
+#Clear Command History
+history -c && history -w
+
+#shutdown and convert to template
+shutdown now

~host-setups/host-git-setup.txt

@@ -0,0 +1,44 @@
+------gitea token------
+#get from --> gitea --> site administration --> actions --> runners
+
+------install and register runner on host------
+# Download binary
+wget https://dl.gitea.com/act_runner/0.2.11/act_runner-0.2.11-linux-amd64 -O /usr/local/bin/act_runner
+chmod +x /usr/local/bin/act_runner
+
+# Register (Replace <URL> and <TOKEN>)
+# The --labels "pve:host" is key for your non-docker workflow
+act_runner register --no-interactive --instance https://git.mapletree.email --token 4aLQr2M0Ox5aUdMaOoSIoHhyUMgCWCrFNuKCFX5l --name pve1-runner --labels "pve1:host"
+
+------Create and move config files------
+# Create the directory first
+mkdir -p /etc/gitea-runner
+
+# Generate the base config
+/usr/local/bin/act_runner generate-config > /etc/gitea-runner/config.yaml
+
+# Move the hidden .runner file (created during registration)
+mv .runner /etc/gitea-runner/
+
+------Create service File------
+
+cat <<EOF > /etc/systemd/system/gitea-runner.service
+[Unit]
+Description=Gitea Actions runner
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/act_runner daemon --config /etc/gitea-runner/config.yaml
+WorkingDirectory=/etc/gitea-runner
+User=root
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+------start and enable------
+
+systemctl daemon-reload
+systemctl enable --now gitea-runner

~host-setups/keepalived-setup.txt

@@ -0,0 +1,33 @@
+[[stack]]
+name = "pve1lxc6-keepalived"
+[stack.config]
+server = "pve1-lxc6"
+linked_repo = "mapletree-pve1lxc6"
+run_directory = "/docker/keepalived"
+file_paths = [
+  "/etc/komodo/repos/mapletree-pve1lxc6/keepalived/docker-compose.yml"
+]
+pre_deploy.command = """
+  # Add# 1. Create Directory
+mkdir -p /docker/keepalived/config
+mkdir -p /docker/keepalived/checks
+
+# 2. Copy Templates from Checked-out Repo
+# (Overwrites existing files, which is what we want)
+cp /etc/komodo/repos/mapletree-pve1lxc6/keepalived/keepalived.conf.tpl /docker/keepalived/config/keepalived.conf
+cp /etc/komodo/repos/mapletree-pve1lxc6/keepalived/check_komodo.sh /docker/keepalived/checks/check_komodo.sh
+
+# 3. Permissions
+chmod +x /docker/keepalived/checks/check_komodo.sh
+
+# 4. Inject Variables (Directly modifying the file on Host)
+# Since we mount the DIRECTORY, the container will see these changes.
+sed -i "s/{{STATE}}/BACKUP/g" /docker/keepalived/config/keepalived.conf
+sed -i "s/{{PRIORITY}}/100/g" /docker/keepalived/config/keepalived.conf
+sed -i "s/{{PEER}}/172.16.201.206/g" /docker/keepalived/config/keepalived.conf
+sed -i "s/{{PASSWORD}}/HAPass22/g" /docker/keepalived/config/keepalived.conf
+"""
+environment = """
+
+
+"""

~host-setups/move-komodo-postgres.txt

@@ -0,0 +1,265 @@
+----first host setup-------
+
+##backup the db
+cd /docker/management
+mkdir ./backup
+chmod 777 ./backup 
+docker run --rm --network container:ferretdb -v $(pwd)/backup:/backup mongo:6.0 mongodump --uri="mongodb://admin:admin@127.0.0.1:27017/komodo" --out=/backup
+
+# CRITICAL SAFETY CHECK:
+# Ensure these files actually exist and have size > 0
+ls -lh ./backup/komodo/
+# If you see empty folders or no files, STOP. Do not proceed to step 2.
+
+##destroy the containers
+docker compose down -v
+
+##change compose to this
+services:
+  mongo:
+    image: mongo:7.0
+    container_name: mongo
+    restart: always
+    network_mode: host
+    command: ["--replSet", "rs0", "--bind_ip_all", "--port", "27017"]
+    volumes:
+      - /docker/management/mongodb:/data/db
+
+  komodo:
+    # REVERT TO MOGHTECH
+    image: ghcr.io/moghtech/komodo-core:latest
+    container_name: komodo
+    cpus: 2.0
+    mem_limit: "2048m"
+    mem_reservation: "512m"
+    network_mode: host
+    env_file:
+      - /docker/management/.env
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /root/.ssh:/home/komodo/.ssh:ro
+      - /docker/management/komodo/config:/config
+      - /docker/management/komodo/backups:/backups
+      - /docker/management/komodo/core-etc:/etc/komodo
+      - /docker/management/komodo/var:/var/lib/komodo
+      - /docker/management/komodo/repo-cache:/repo-cache
+      - /docker:/docker
+    environment:
+      # CHANGE TO 'URI'
+      # This tells Komodo: "Here is the full connection string, don't try to parse commas."
+      - KOMODO_DATABASE_URI=mongodb://127.0.0.1:27017/komodo?directConnection=true&replicaSet=rs0
+
+      # Explicitly unset ADDRESS to avoid conflicts
+      - KOMODO_DATABASE_ADDRESS=
+    depends_on:
+      - mongo
+    restart: unless-stopped
+    labels:
+      - "komodo.skip=true"
+
+    
+## up the mongodb service
+docker compose up -d mongo
+
+## restore the data to the container
+docker run --rm --network host -v $(pwd)/backup:/backup mongo:6.0 mongorestore --uri="mongodb://127.0.0.1:27017/komodo" /backup/komodo
+
+##make sure that mongo knows it's the primary
+docker exec -it mongo mongosh --eval 'rs.initiate({_id: "rs0", members: [{ _id: 0, host: "172.16.201.201:27017" }]})'
+
+##make sure the framework is done
+##make sure mongo connection string is active in compose 
+docker compose up -d komodo
+
+##open gui to confirm it works
+
+##restore the data into the framework in mongo
+docker run --rm --user 0:0 --network host -v $(pwd)/backup:/backup mongo:6.0 mongorestore --uri="mongodb://127.0.0.1:27017/komodo?replicaSet=rs0" /backup/komodo
+
+##compose up the komodo
+docker compose up -d komodo
+
+##validate
+
+----second host base setup-------
+
+##get the env setup
+mkdir /docker/managment/mongo
+
+##deploy the same docker-compose.yml as above
+##but only start the mongo for now
+docker compose up -d mongo
+
+----first host replica setup-------
+
+##get into the original mongo
+docker exec -it mongo mongosh
+
+##add the new mongo member
+rs.add("172.16.201.106")
+
+##check status
+rs.status()
+
+
+----komodo redeploy-------
+
+##change komodo compose file on both to have this as DB URI
+environment:
+      # LIST BOTH IPs
+      # This allows the driver to failover automatically.
+      - KOMODO_DATABASE_URI=mongodb://172.16.201.206:27017,172.16.201.106:27017/komodo?replicaSet=rs0
+
+##redeploy the original komodo
+docker compose down komodo && docker compose up -d komodo
+
+##deploy the secondary
+docker compose up -d komodo
+
+----second host replica check-------
+
+##get into the second mongo
+docker exec -it mongo mongosh
+
+##do a secondary check
+db.getMongo().setReadPref('secondary')
+
+#create a stack called "replica-test"
+
+##check that the data exists
+use komodo
+db.Stack.find({name: "replica-test"})
+
+----add arbiter-------
+
+##already deployed on pve1-lxc2 - redeploy
+-->destroy the container
+-->delete the mongo-arbiter folder in pve-lxc2/docker
+--> use komodo to redeploy the mongo-arbiter
+
+##do the following on whichever mongo is primary to add the arbiter
+docker exec -it mongo mongosh
+db.adminCommand({
+  setDefaultRWConcern: 1,
+  defaultWriteConcern: { w: 1 }
+})
+##add the arbiter running on pve1-lxc2
+rs.addArb("172.16.201.102:27017")
+
+##check DB HA
+rs.status()
+
+----now do the lsyncd-------
+
+##make sure pve1-lxc6 has rsync
+apt update && apt install rsync -y
+
+##do on PVE2-lxc6
+##install lsyncd and rsync
+apt update && apt install lsyncd rsync -y
+
+##generate keys
+ssh-keygen -t rsa -b 4096
+
+##lock down the keys - ensure root owns it
+chown root:root /etc/ssh/ssh_config.d/20-systemd-ssh-proxy.conf
+
+##lock down the keys - Ensure only root can write to it
+chmod 644 /etc/ssh/ssh_config.d/20-systemd-ssh-proxy.conf
+
+##May have to do a keygen on the target host
+ssh-keygen -A
+
+##send key to pve1-lxc6
+ssh-copy-id root@172.16.201.101
+
+## test
+ssh root@172.16.201.101
+
+##if it asks for password do this on the other lxc
+chmod 700 /root/.ssh
+chmod 600 /root/.ssh/authorized_keys
+chown -R root:root /root/.ssh
+
+##create config file
+mkdir /etc/lsyncd
+nano /etc/lsyncd/lsyncd.conf.lua
+
+##paste this
+settings {
+    logfile = "/var/log/lsyncd/lsyncd.log",
+    statusFile = "/var/log/lsyncd/lsyncd.status",
+    nodaemon = false,
+}
+
+-- The Sync Configuration
+sync {
+    default.rsync,
+    
+    -- The folder on THIS server to watch
+    -- CHECK: Is this mapped to /repo in your compose?
+    source = "/repo/mapletree-pve2lxc6/management/komodo",
+
+    -- The Destination
+    target = "root@172.16.201.106:/repo/mapletree-pve1lxc6/management/komodo",
+
+    -- Exclude temporary files
+    exclude = { '.git', '*.tmp', '*.bak' },
+
+    -- Rsync Options (Archive mode, Compress, Delete files on target if deleted on source)
+    rsync = {
+        archive = true,
+        compress = true,
+        verbose = true,
+        _extra = { "--omit-dir-times" } 
+    }
+}
+
+##make sure the service is gunna work
+mkdir -p /var/log/lsyncd
+touch /var/log/lsyncd/lsyncd.log
+touch /var/log/lsyncd/lsyncd.status
+
+set the stack compose variables to (respectively):
+REPO_ROOT=/repo/mapletree-pve2lxc6/management/komodo
+REPO_ROOT=/repo/mapletree-pve1lxc6/management/komodo
+
+##change the komodo compose block to include the repo...
+volumes:
+  - ${REPO_ROOT}:/repo
+  
+##redeploy the komodos
+--> may have to do this cli
+
+##start the sync
+systemctl restart lsyncd
+systemctl status lsyncd
+systemctl enable --now lsyncd
+
+##the check (run on pve2 - 2 sessions)
+tail -f /var/log/lsyncd/lsyncd.log
+touch /repo/mapletree-pve2lxc6/management/komodo/sync_test.txt
+
+##check on PVE1
+ls -l /repo/mapletree-pve1lxc6/management/komodo/
+
+
+----now do the keepalived setup-------
+
+##Make sure the LXC's are configured for the IP:
+echo "net.ipv4.ip_nonlocal_bind=1" >> /etc/sysctl.conf
+sysctl -p
+
+##make sure the keepalive folder is created and mounted into the periphery
+mkdir /docker/keepalived || true
+nano /root/periphery/docker-compose.yml
+
+##don't forget to compose down/up the periphery
+docker compose down && docker compose up -d
+
+##deploy the containers 
+##make sure the vars are done
+--> in Komodo
+
+
+