# docker 
$ sudo apt-get update

$ sudo apt-get install apt-transport-https ca-certificates curl gnupg lsb-release

# 添加软件源的 GPG 密钥
$ curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
# 官方源
# $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

# 向 sources.list 中添加 Docker 软件源
$ echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
# 官方源
# $ echo \
#   "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
#   $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

# 更新 apt 软件包缓存，并安装 docker-ce
$ sudo apt-get update
$ sudo apt-get install docker-ce docker-ce-cli containerd.io

# 更换阿里云镜像源
$ sudo mkdir -p /etc/docker
$ sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": [
      "https://y9wyvozl.mirror.aliyuncs.com",
      "https://hub-mirror.c.163.com",
      "https://mirror.baidubce.com"]
}
EOF
$ sudo systemctl daemon-reload
$ sudo systemctl restart docker

# docker system df
[root@node1 ~]# docker system df
TYPE            TOTAL     ACTIVE    SIZE      RECLAIMABLE
Images          56        11        13.42GB   12.28GB (91%)
Containers      19        19        33.94kB   0B (0%)
Local Volumes   0         0         0B        0B
Build Cache     0         0         0B        0B

# docker system prune命令可以用于清理磁盘，删除关闭的容器、无用的数据卷和网络，以及 dangling 镜像(即无 tag 的镜像)。
# docker system prune -a命令清理得更加彻底，可以将没有容器使用 Docker 镜像都删掉。

$ find /var/lib/docker/overlay2/ |grep 'gdb$'
# /var/lib/docker/overlay2/e532b7032e978fe3b7a0c221974aa3d76739a5725908436e45b286faaf1f39fd/diff/usr/share/gdb
$ find /var/lib/docker |grep gdb$ | awk -F/ '{print $6}' | uniq | sort
# 05fbe99efaa43a14a6e5b1fd75b15c390087990432a69a502e5c43ae2f316d9d
# 0c22f1324df153584e04ca822e3161ec29dbef2b5db931de7a2bd169d8e33297

# 查询退出的容器
$ docker ps -q | awk '{if (NR>1){print $1":"$2}}' | xargs docker inspect --format '{{.State.Pid}}, {{.Id}}, {{.Name}}, {{.GraphDriver.Data.WorkDir}}' | grep "e532b7032e978fe3b7a0c221974aa3d76739a5725908436e45b286faaf1f39fd"

# 查询所有的容器
$ docker ps -a | awk '{if (NR>1){print $1":"$2}}'| xargs docker inspect --format '{{.State.Pid}}, {{.Id}}, {{.Name}}, {{.GraphDriver.Data}}' | grep "e532b7032e978fe3b7a0c221974aa3d76739a5725908436e45b286faaf1f39fd"

# 查询所有镜像
$ docker image ls | awk '{if (NR>1){print $1":"$2}}' | xargs docker inspect --format '{{.RepoTags}}, {{.GraphDriver.Data}}'

# 查询所有底层镜像带有gdb工具的镜像
$ docker image ls | awk '{if (NR>1){print $1":"$2}}' | xargs docker inspect --format '{{.RepoTags}}, {{.GraphDriver.Data}}' | grep -E $(echo $(find /var/lib/docker |grep /gdb$ | awk -F/ '{print $6}' | uniq | sort) | sed 's/ /|/g') | awk -F, '{print $1}'

# Install the qemu packages
sudo apt-get install qemu binfmt-support qemu-user-static

# This step will execute the registering scripts
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes

# Testing the emulation environment
docker run --rm -t arm64v8/ubuntu uname -m

# link
https://www.stereolabs.com/docs/docker/building-arm-container-on-x86/
https://devopstales.github.io/linux/running_and_building_multi_arch_containers/

# k3s
$ curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=latest sh -

# cat /etc/rancher/k3s/k3s.yaml

# openebs
# helm 安装
# tips: 未安装helm的需要先根据 4.1 安装helm
$ helm repo add openebs https://openebs.github.io/charts
$ helm repo update
$ helm install openebs --namespace openebs openebs/openebs --create-namespace

# k3s 使用helm install异常
# Error: Kubernetes cluster unreachable: Get "http://localhost:8080/version?timeout=32s": dial tcp 127.0.0.1:8080: connect: connection refused
# 解决方式
$ mkdir ~/.kube
$ cp /etc/rancher/k3s/k3s.yaml ~/.kube/
$ export KUBECONFIG=/root/.kube/k3s.yaml


# kubectl 安装
$ kubectl apply -f https://openebs.github.io/charts/openebs-operator.yaml

# 设置默认storage-class
# 查询默认sc
kubectl get storageclass

kubectl patch storageclass openebs-hostpath -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
# or
kubectl patch storageclass openebs-device -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

# 安装
$ kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/kubesphere-installer.yaml
   
$ kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.3.0/cluster-configuration.yaml

# 查看安装日志
$ kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f

# 检查pod安装状态
$ kubectl get pod --all-namespaces
# kubectl get pod -A
$ kubectl get svc/ks-console -n kubesphere-system

# 依赖
# 需要安装
$ apt-get install socat
$ apt-get install conntrack

# 建议安装
$ apt-get install ebtables
$ apt-get install ipset

# 确保在安全组中打开了端口 30880，并通过 NodePort (IP:30880) 使用默认帐户和密码 (admin/P@88w0rd) 访问 Web 控制台
url: http://IP:30880

# 可插拔组件
https://kubesphere.io/zh/docs/v3.3/pluggable-components/

# 直接安装
$ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

# 安装kubekey
$ curl -sfL https://get-kk.kubesphere.io | VERSION=v2.2.1 sh -
$ chmod +x ./kk
# 安装kubesphere
$ ./kk create cluster --with-kubesphere v3.3.0

# 验证安装结果
$ kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f

# 安装kubernetes
$ ./kk create cluster --with-kubernetes v1.23.0

/usr/local/bin/k3s-uninstall.sh

./kk delete cluster