triton - how to set the batch size: dynamic versus predefined

predefined:-

name: "kps"
platform: "tensorrt_plan"
max_batch_size: 16
dynamic_batching {
  preferred_batch_size: [ 1,2,4,8,16 ]
  max_queue_delay_microseconds: 50
}

instance_group [
    {
      count: 16
      kind: KIND_GPU
      gpus: [ 0 ]
    }
]

input [
  {
    name: "input"
    data_type: TYPE_FP32
    format: FORMAT_NCHW
    dims: [ 3, 64, 64 ]
    reshape { shape: [ 3, 64, 64 ] }
  }
]
output [
  {
    name: "output1"
    data_type: TYPE_FP32
    dims: [ 1]
  },
  {
    name: "output2"
    data_type: TYPE_INT32
    dims: [ 1 ]
  },
  {
    name: "output3"
    data_type: TYPE_FP32
    dims: [ 8,64 ]
  }
]



version_policy: { all { } }
 

 

dynamic:-

platform: "tensorrt_plan"
max_batch_size: 0

instance_group [
    {
      count: 16
      kind: KIND_GPU
      gpus: [ 0 ]
    }
]


input [
  {
    name: "images"
    data_type: TYPE_FP32
    dims: [ -1,3, 640, 640]
  }
]
output [
  {
    name: "524"
    data_type: TYPE_FP32
    dims: [ -1,3,80,80, 7 ]
  },
      {
    name: "585"
    data_type: TYPE_FP32
    dims: [ -1,3,40,40, 7 ]
  },
  {
    name: "646"
    data_type: TYPE_FP32
    dims: [ -1,3,20,20, 7 ]
  },
  {
    name: "output"
    data_type: TYPE_FP32
    dims: [ -1, 25200, 7 ]
  }
]
~                                                                                                                                                                                                           
~                                                                                                                                                                                                           
~                                                                                                                                                                                                           
~