deploy.yaml

# Setup hivedscheduler by "kubectl apply -f deploy.yaml"
# Notes:
# 1. Currently, K8S Default Scheduler only has one scheduling queue, so one VC's
#    high priority Pods may head-of-line block other VC's low priority Pods from
#    getting scheduling chance, even if the high priority Pods cannot fit into
#    cluster, but the low priority Pods can.
#    This issue is tracked in
#    https://github.com/kubernetes/kubernetes/issues/86373
#
#    To workaround it, here we create one K8S Default Scheduler for each VC, all
#    pointer to the same HiveD Scheduler serving at
#    http://hivedscheduler-service:30096/v1/extender
#
#    So, if a Pod needs to be scheduled by HiveD, it should specify its VC's
#    corresponding schedulerName.
#
# 2. To deploy it automatically based on existing VC configuration, see
#    https://github.com/microsoft/pai/blob/master/src/hivedscheduler/deploy/hivedscheduler.yaml.template
apiVersion: v1
kind: ConfigMap
metadata:
  name: hivedscheduler-config
  namespace: default
data:
  policy.cfg : |
    {
      "kind": "Policy",
      "apiVersion": "v1",
      "extenders": [
        {
          "urlPrefix": "http://hivedscheduler-service:30096/v1/extender",
          "filterVerb": "filter",
          "preemptVerb": "preempt",
          "bindVerb": "bind",
          "enableHttps": false,
          "httpTimeout": 5000000000,
          "nodeCacheCapable": true,
          "ignorable": false,
          "managedResources": [
            {
              "name": "hivedscheduler.microsoft.com/pod-scheduling-enable",
              "ignoredByScheduler": true
            }
          ]
        }
      ]
    }
  hivedscheduler.yaml: |
    webServerAddress: ":30096"
    waitingPodSchedulingBlockMilliSec: 50
    physicalCluster:
      skuTypes:
        K80:
          gpu: 1
          cpu: 5
          memory: 54Gi
      cellTypes:
        K80-NODE:
          childCellType: K80
          childCellNumber: 4
          isNodeLevel: true
        K80-NODE-POOL:
          childCellType: K80-NODE
          childCellNumber: 3
      physicalCells:
      - cellType: K80-NODE-POOL
        cellChildren:
        - cellAddress: node1
        - cellAddress: node2
        - cellAddress: node3

    virtualClusters:
      vc1:
        virtualCells:
        - cellType: K80-NODE-POOL.K80-NODE
          cellNumber: 1
      vc2:
        virtualCells:
        - cellType: K80-NODE-POOL.K80-NODE
          cellNumber: 2
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: hivedscheduler-account
  namespace: default
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: hivedscheduler-role-binding
  namespace: default
subjects:
  - kind: ServiceAccount
    name: hivedscheduler-account
    namespace: default
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: hivedscheduler-hs
  namespace: default
spec:
  serviceName: hivedscheduler-hs
  selector:
    matchLabels:
      app: hivedscheduler-hs
  replicas: 1
  template:
    metadata:
      labels:
        app: hivedscheduler-hs
    spec:
      serviceAccountName: hivedscheduler-account
      containers:
      - name: hivedscheduler
        image: hivedscheduler/hivedscheduler
        env:
        - name: CONFIG
          value: /hivedscheduler-config/hivedscheduler.yaml
        command: [
          "bash", "-c",
          "./start.sh"]
        volumeMounts:
        - name: hivedscheduler-config
          mountPath: /hivedscheduler-config
      volumes:
      - name: hivedscheduler-config
        configMap:
          name: hivedscheduler-config
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: hivedscheduler-ds-vc1
  namespace: default
spec:
  serviceName: hivedscheduler-ds-vc1
  selector:
    matchLabels:
      app: hivedscheduler-ds-vc1
  replicas: 1
  template:
    metadata:
      labels:
        app: hivedscheduler-ds-vc1
    spec:
      serviceAccountName: hivedscheduler-account
      containers:
      - name: defaultscheduler
        image: gcr.io/google_containers/kube-scheduler:v1.14.2
        command: [
          "sh", "-c",
          "echo \"apiVersion: kubescheduler.config.k8s.io/v1alpha1\" >> config.yaml &&
           echo \"kind: KubeSchedulerConfiguration\" >> config.yaml &&
           echo \"schedulerName: hivedscheduler-ds-vc1\" >> config.yaml &&
           echo \"disablePreemption: false\" >> config.yaml &&
           echo \"percentageOfNodesToScore: 100\" >> config.yaml &&
           echo \"algorithmSource:\" >> config.yaml &&
           echo \"  policy:\" >> config.yaml &&
           echo \"    configMap:\" >> config.yaml &&
           echo \"      name: hivedscheduler-config\" >> config.yaml &&
           echo \"      namespace: default\" >> config.yaml &&
           echo \"leaderElection:\" >> config.yaml &&
           echo \"  leaderElect: false\" >> config.yaml &&
           /usr/local/bin/kube-scheduler
           --config=config.yaml
           --feature-gates=PodPriority=true
           --leader-elect=false
           --v=4"]
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: hivedscheduler-ds-vc2
  namespace: default
spec:
  serviceName: hivedscheduler-ds-vc2
  selector:
    matchLabels:
      app: hivedscheduler-ds-vc2
  replicas: 1
  template:
    metadata:
      labels:
        app: hivedscheduler-ds-vc2
    spec:
      serviceAccountName: hivedscheduler-account
      containers:
      - name: defaultscheduler
        image: gcr.io/google_containers/kube-scheduler:v1.14.2
        command: [
          "sh", "-c",
          "echo \"apiVersion: kubescheduler.config.k8s.io/v1alpha1\" >> config.yaml &&
           echo \"kind: KubeSchedulerConfiguration\" >> config.yaml &&
           echo \"schedulerName: hivedscheduler-ds-vc2\" >> config.yaml &&
           echo \"disablePreemption: false\" >> config.yaml &&
           echo \"percentageOfNodesToScore: 100\" >> config.yaml &&
           echo \"algorithmSource:\" >> config.yaml &&
           echo \"  policy:\" >> config.yaml &&
           echo \"    configMap:\" >> config.yaml &&
           echo \"      name: hivedscheduler-config\" >> config.yaml &&
           echo \"      namespace: default\" >> config.yaml &&
           echo \"leaderElection:\" >> config.yaml &&
           echo \"  leaderElect: false\" >> config.yaml &&
           /usr/local/bin/kube-scheduler
           --config=config.yaml
           --feature-gates=PodPriority=true
           --leader-elect=false
           --v=4"]
---
apiVersion: v1
kind: Service
metadata:
  name: hivedscheduler-service
spec:
  selector:
    app: hivedscheduler-hs
  type: NodePort
  ports:
  - protocol: TCP
    targetPort: 30096
    port: 30096
    nodePort: 30096