apiVersion: serving.kserve.io/v1alpha1

kind: ServingRuntime

labels:

  opendatahub.io/dashboard: "true"

metadata:

  annotations:

    openshift.io/display-name: sglang 0.4.3

  name: sglang-0.4.3

spec:

  builtInAdapter:

    modelLoadingTimeoutMillis: 90000

  containers:

    - image: quay.io/qxu/sglang:v0.4.3.post2-cu125

      command:

        - python3

        - -m

        - sglang.launch_server

      args:

        - --model

        - /mnt/models/

        - --served-model-name

        - DeepSeek-V3

        - --tp

        - "8"

        - --trust-remote-code

        - --port

        - "8080"

        - --quantization

        - fp8

        - --mem-fraction-static

        - "0.90"

        - --context-length

        - "64000"

        - --enable-metrics

        - --enable-torch-compile

        - --log-requests

      name: kserve-container

      ports:

        - containerPort: 8080

          name: http1

          protocol: TCP

  multiModel: false

  supportedModelFormats:

    - autoSelect: true

      name: pytorch

kind: Service

apiVersion: v1

metadata:

  name: deepseek-v3-nodeport

  labels:

    app: isvc.deepseek-v3-predictor

    component: predictor

    opendatahub.io/dashboard: 'true'

    serving.kserve.io/inferenceservice: deepseek-v3

spec:

  ports:

    - protocol: TCP

      port: 8080

      targetPort: 8080

      nodePort: 30001

  type: NodePort

  selector:

    app: isvc.deepseek-v3-predictor