Tips to build Qemu S390x images

1. Connect to a build machine

ssh root@ip

2. Clone the operator

“git clone https://github.com/prb112/operator.git“`

3. Install qemu and buildah and podman-docker

yum install -y qemu-kvm buildah podman-docker

/usr/bin/docker run --rm --privileged tonistiigi/binfmt:latest --install all
Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg.
✔ docker.io/tonistiigi/binfmt:latest
Trying to pull docker.io/tonistiigi/binfmt:latest...
Getting image source signatures
Copying blob e9c608ddc3cb done  
Copying blob 8d4d64c318a5 done  
Copying config 354472a378 done  
Writing manifest to image destination
Storing signatures
installing: arm64 OK
installing: arm OK
installing: ppc64le OK
installing: mips64 OK
installing: riscv64 OK
installing: mips64le OK
installing: s390x OK
{
  "supported": [
    "linux/amd64",
    "linux/arm64",
    "linux/riscv64",
    "linux/ppc64le",
    "linux/s390x",
    "linux/386",
    "linux/mips64le",
    "linux/mips64",
    "linux/arm/v7",
    "linux/arm/v6"
  ],
  "emulators": [
    "kshcomp",
    "qemu-aarch64",
    "qemu-arm",
    "qemu-mips64",
    "qemu-mips64el",
    "qemu-ppc64le",
    "qemu-riscv64",
    "qemu-s390x"
  ]
}

/usr/bin/buildah bud --arch s390x -f $(pwd)/build/Dockerfile --format docker --tls-verify=true -t op:v0.1.1-linux-s390x $(pwd)/

1. Update Hosts

9.0.90.0 ocp4daily70.ibm.com
9.0.90.1 ocp4daily98.ibm.com

2. Setup the Subscription Manager

set +o history
export rhel_subscription_username="rhn-ee-xxxxx"
export rhel_subscription_password="xxxxx"
set -o history
subscription-manager register --username="${rhel_subscription_username}" --password="${rhel_subscription_password}"
subscription-manager refresh

3. Disable the swap

sudo swapoff -a

4. Install the libraries

yum install -y podman podman-remote socat runc

5. Install the cri-o package

rpm -ivh https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.25:/1.25.0/Fedora_36/ppc64le/cri-o-1.25.0-2.1.fc36.ppc64le.rpm

6. Enable podman socket

systemctl enable --now podman.socket

7. Enable crio service

sudo systemctl enable crio
sudo systemctl start crio

8. Disable selinux

sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

9. Download Release

export RELEASE=1.25
sudo curl -L --remote-name-all https://dl.k8s.io/v1.25.2/bin/linux/ppc64le/{kubeadm,kubelet,kubectl}
sudo chmod +x {kubeadm,kubelet,kubectl}

15. Move files to /bin

mv kube* /bin/

16. Add kubelet.service

RELEASE_VERSION="v0.14.0"
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" | sudo tee /etc/systemd/system/kubelet.service
sudo mkdir -p /etc/systemd/system/kubelet.service.d
curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:${DOWNLOAD_DIR}:g" | sudo tee /etc/systemd/system/kubelet.service.d/10-kubeadm.conf

17. Enable and start service

systemctl enable --now kubelet
systemctl start kubelet

18. Update the cgroup settings

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

19. Load the modules

sudo modprobe overlay
sudo modprobe br_netfilter

20. sysctl params required by setup, params persist across reboots

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

21. Apply sysctl params without reboot

sudo sysctl --system

22. Install libnetfilter and conntrack-tools

rpm -ivh http://mirror.stream.centos.org/9-stream/AppStream/ppc64le/os/Packages/libnetfilter_queue-1.0.5-1.el9.ppc64le.rpm
rpm -ivh http://mirror.stream.centos.org/9-stream/AppStream/ppc64le/os/Packages/libnetfilter_cttimeout-1.0.0-19.el9.ppc64le.rpm
rpm -ivh http://mirror.stream.centos.org/9-stream/AppStream/ppc64le/os/Packages/libnetfilter_cthelper-1.0.0-22.el9.ppc64le.rpm
rpm -ivh http://mirror.stream.centos.org/9-stream/AppStream/ppc64le/os/Packages/conntrack-tools-1.4.5-15.el9.ppc64le.rpm

23. Copy Kubelet

cp /bin/kubelet /kubelet

24. Edit crio.conf

/etc/crio/crio.conf

conmon_cgroup = "pod"
cgroup_manager = "systemd"

25. Add the plugins:

curl -O https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-ppc64le-v1.1.1.tgz -L
cp cni-plugins-linux-ppc64le-v1.1.1.tgz /opt/cni/bin
cd /opt/cni/bin
tar xvfz cni-plugins-linux-ppc64le-v1.1.1.tgz 
chmod +x /opt/cni/bin/*
cd ~
systemctl restart crio kubelet

26. Download crictl

curl -L --remote-name-all https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.25.0/crictl-v1.25.0-linux-ppc64le.tar.gz
tar xvfz crictl-v1.25.0-linux-ppc64le.tar.gz
chmod +x crictl
mv crictl /bin

27. Create the kubeadm

kubeadm init --cri-socket=unix:///var/run/crio/crio.sock --pod-network-cidr=192.168.0.0/16

28. Setup the configuration

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

29. Manually copy over the .kube/config over to the worker node and do a kubeadm reset

30. Download https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

31. Edit the containers to point to the right instance, per the notes in the yaml to the ppc64le manifests

32. Update net-conf.json

  net-conf.json: |
    {
      "Network": "192.168.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }

33. Join the Cluster

kubeadm join 9.0.90.1:6443 --token xbp7gy.9eem3bta75v0ccw8 \
        --discovery-token-ca-cert-hash sha256:a822342f231db2e730559b4962325a2c2c685d7fc440ae41987e123da47f9118

34. Add role to the workers

kubectl label node ocp4daily70.ibm.com node-role.kubernetes.io/worker=worker