From 1eee7e49402639b3b1beab18c798c98bafeeb5a8 Mon Sep 17 00:00:00 2001 From: carbon Date: Fri, 31 May 2024 11:41:59 +0800 Subject: [PATCH] add cvibuilder commit 4309f2a649fc7cfe7160389d52a81c469dbdd7bc Author: sophgo-forum-service Date: Mon May 13 14:11:17 2024 +0800 [feat] cvibuilder opensource for cv18xx soc. - 60a1a5 --- .version/2024-05-07.md | 36 +++--- .version/2024-05-31.md | 19 +++ cvibuilder/.gitignore | 3 + cvibuilder/CMakeLists.txt | 29 +++++ cvibuilder/README.md | 2 + cvibuilder/proto/cvimodel.fbs | 215 +++++++++++++++++++++++++++++++++ cvibuilder/proto/parameter.fbs | 46 +++++++ 7 files changed, 332 insertions(+), 18 deletions(-) create mode 100644 .version/2024-05-31.md create mode 100644 cvibuilder/.gitignore create mode 100644 cvibuilder/CMakeLists.txt create mode 100644 cvibuilder/README.md create mode 100644 cvibuilder/proto/cvimodel.fbs create mode 100644 cvibuilder/proto/parameter.fbs diff --git a/.version/2024-05-07.md b/.version/2024-05-07.md index acb5aaf56..01a57cb4f 100644 --- a/.version/2024-05-07.md +++ b/.version/2024-05-07.md @@ -1,18 +1,18 @@ - -| Package | PATH | URL | Branch | Commit | -|:---------------------------|:------------------------------------|:---------------------------------------------------------|:--------------|:-------------| -| root dir | | https://github.com/sophgo/sophpi.git | sg200x-evb | d1edce7 | -| build | build | https://github.com/sophgo/build.git | sg200x-dev | 985b304 | -| fsbl | fsbl | https://github.com/sophgo/fsbl.git | sg200x-dev | 426d1e7 | -| opensbi | opensbi | https://github.com/sophgo/opensbi.git | sg200x-dev | 216793f | -| u-boot-2021.10 | u-boot-2021.10 | https://github.com/sophgo/u-boot-2021.10.git | sg200x-dev | 396f62108c | -| linux_5.10 | linux_5.10 | https://github.com/sophgo/linux_5.10.git | sg200x-dev | 25faa0d62fb5 | -| ramdisk | ramdisk | https://github.com/sophgo/ramdisk.git | sg200x-dev | 9a529ad | -| middleware | middleware | https://github.com/sophgo/middleware.git | sg200x-dev | 07726f4 | -| SensorSupportList | middleware/v2/component/isp | https://github.com/sophgo/SensorSupportList.git | cv18xx-v4.1.x | caade0b | -| isp_tuning | isp_tuning | https://github.com/sophgo/isp_tuning.git | sg200x-dev | 6504909 | -| osdrv | osdrv | https://github.com/sophgo/osdrv.git | sg200x-dev | 0c88923 | -| oss | oss | https://github.com/sophgo/oss.git | master | 59b6a54 | -| freertos | freertos | https://github.com/sophgo/freertos.git | sg200x-dev | d3d075910 | -| FreeRTOS-Kernel | freertos/Source | https://github.com/sophgo/FreeRTOS-Kernel.git | sg200x-dev | d52c1b6e6 | -| Lab-Project-FreeRTOS-POSIX | freertos/Source/FreeRTOS-Plus-POSIX | https://github.com/sophgo/Lab-Project-FreeRTOS-POSIX.git | sg200x-dev | 5042bfd | + +| Package | PATH | URL | Branch | Commit | +|:---------------------------|:------------------------------------|:---------------------------------------------------------|:--------------|:-------------| +| root dir | | https://github.com/sophgo/sophpi.git | sg200x-evb | d1edce7 | +| build | build | https://github.com/sophgo/build.git | sg200x-dev | 985b304 | +| fsbl | fsbl | https://github.com/sophgo/fsbl.git | sg200x-dev | 426d1e7 | +| opensbi | opensbi | https://github.com/sophgo/opensbi.git | sg200x-dev | 216793f | +| u-boot-2021.10 | u-boot-2021.10 | https://github.com/sophgo/u-boot-2021.10.git | sg200x-dev | 396f62108c | +| linux_5.10 | linux_5.10 | https://github.com/sophgo/linux_5.10.git | sg200x-dev | 25faa0d62fb5 | +| ramdisk | ramdisk | https://github.com/sophgo/ramdisk.git | sg200x-dev | 9a529ad | +| middleware | middleware | https://github.com/sophgo/middleware.git | sg200x-dev | 07726f4 | +| SensorSupportList | middleware/v2/component/isp | https://github.com/sophgo/SensorSupportList.git | cv18xx-v4.1.x | caade0b | +| isp_tuning | isp_tuning | https://github.com/sophgo/isp_tuning.git | sg200x-dev | 6504909 | +| osdrv | osdrv | https://github.com/sophgo/osdrv.git | sg200x-dev | 0c88923 | +| oss | oss | https://github.com/sophgo/oss.git | master | 59b6a54 | +| freertos | freertos | https://github.com/sophgo/freertos.git | sg200x-dev | d3d075910 | +| FreeRTOS-Kernel | freertos/Source | https://github.com/sophgo/FreeRTOS-Kernel.git | sg200x-dev | d52c1b6e6 | +| Lab-Project-FreeRTOS-POSIX | freertos/Source/FreeRTOS-Plus-POSIX | https://github.com/sophgo/Lab-Project-FreeRTOS-POSIX.git | sg200x-dev | 5042bfd | diff --git a/.version/2024-05-31.md b/.version/2024-05-31.md new file mode 100644 index 000000000..cf463e248 --- /dev/null +++ b/.version/2024-05-31.md @@ -0,0 +1,19 @@ + +| Package | PATH | URL | Branch | Commit | +|:---------------------------|:------------------------------------|:---------------------------------------------------------|:--------------|:-------------| +| root dir | | https://github.com/sophgo/sophpi.git | sg200x-evb | d1edce7 | +| build | build | https://github.com/sophgo/build.git | sg200x-dev | 985b304 | +| fsbl | fsbl | https://github.com/sophgo/fsbl.git | sg200x-dev | 426d1e7 | +| opensbi | opensbi | https://github.com/sophgo/opensbi.git | sg200x-dev | 216793f | +| u-boot-2021.10 | u-boot-2021.10 | https://github.com/sophgo/u-boot-2021.10.git | sg200x-dev | 396f62108c | +| linux_5.10 | linux_5.10 | https://github.com/sophgo/linux_5.10.git | sg200x-dev | 25faa0d62fb5 | +| ramdisk | ramdisk | https://github.com/sophgo/ramdisk.git | sg200x-dev | 9a529ad | +| middleware | middleware | https://github.com/sophgo/middleware.git | sg200x-dev | 07726f4 | +| SensorSupportList | middleware/v2/component/isp | https://github.com/sophgo/SensorSupportList.git | cv18xx-v4.1.x | caade0b | +| isp_tuning | isp_tuning | https://github.com/sophgo/isp_tuning.git | sg200x-dev | 6504909 | +| osdrv | osdrv | https://github.com/sophgo/osdrv.git | sg200x-dev | 0c88923 | +| oss | oss | https://github.com/sophgo/oss.git | master | 59b6a54 | +| freertos | freertos | https://github.com/sophgo/freertos.git | sg200x-dev | d3d075910 | +| FreeRTOS-Kernel | freertos/Source | https://github.com/sophgo/FreeRTOS-Kernel.git | sg200x-dev | d52c1b6e6 | +| Lab-Project-FreeRTOS-POSIX | freertos/Source/FreeRTOS-Plus-POSIX | https://github.com/sophgo/Lab-Project-FreeRTOS-POSIX.git | sg200x-dev | 5042bfd | +| cvibuilder | cvibuilder | https://github.com/sophgo/cvibuilder.git | sg200x-dev | 4309f2a | diff --git a/cvibuilder/.gitignore b/cvibuilder/.gitignore new file mode 100644 index 000000000..8b84bd817 --- /dev/null +++ b/cvibuilder/.gitignore @@ -0,0 +1,3 @@ +build +*.pyc +*.cm diff --git a/cvibuilder/CMakeLists.txt b/cvibuilder/CMakeLists.txt new file mode 100644 index 000000000..44e70cf76 --- /dev/null +++ b/cvibuilder/CMakeLists.txt @@ -0,0 +1,29 @@ +cmake_minimum_required(VERSION 3.1.0) + +project(cvibuilder C CXX) + +if (NOT DEFINED FLATBUFFERS_PATH) + message(FATAL_ERROR "Please set FLATBUF_PATH") +endif() +message(STATUS "FLATBUFFERS_PATH: ${FLATBUFFERS_PATH}") + +function(compile_fbs) + foreach(fbs_file IN LISTS ARGN) + get_filename_component(name ${fbs_file} NAME_WE) + set(target_header ${CMAKE_CURRENT_BINARY_DIR}/include/cvibuilder/${name}_generated.h) + message(STATUS "To compile fbs files ${fbs_file} for ${target_header}") + add_custom_command(OUTPUT ${target_header} + DEPENDS ${fbs_file} + COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/include/cvibuilder + COMMAND ${FLATBUFFERS_PATH}/bin/flatc -o ${CMAKE_CURRENT_BINARY_DIR}/include/cvibuilder --cpp ${fbs_file} + ) + set(targets ${targets} ${target_header}) + endforeach() + add_custom_target(cvibuilder ALL DEPENDS ${targets}) + install(FILES ${targets} DESTINATION include/cvibuilder/) +endfunction() + +compile_fbs( + ${CMAKE_CURRENT_SOURCE_DIR}/proto/cvimodel.fbs + ${CMAKE_CURRENT_SOURCE_DIR}/proto/parameter.fbs +) diff --git a/cvibuilder/README.md b/cvibuilder/README.md new file mode 100644 index 000000000..a33cc3562 --- /dev/null +++ b/cvibuilder/README.md @@ -0,0 +1,2 @@ +# cvibuilder + diff --git a/cvibuilder/proto/cvimodel.fbs b/cvibuilder/proto/cvimodel.fbs new file mode 100644 index 000000000..cf2292f94 --- /dev/null +++ b/cvibuilder/proto/cvimodel.fbs @@ -0,0 +1,215 @@ +namespace cvi.model; + +/// +/// General Descriptions +/// Model: +/// A model is generated from one NN model, it contains one or more +/// `Segment`s, each `Segment` could be either a TPU segment or a CPU +/// segment. A Model takes one or more `Tensor`s as input, and produces +/// one or more `Tensor's as output. Along with `Segment`s, a list of +/// `Memory` and a list of `Tensor` are provided. +/// Segment: +/// A segment is a program running on a certain type of engine (`TPU` or +/// `CPU`), which takes a list of `Tensor`s as input, and produces a list +/// of `Tensor`s as output. +/// Memory: +/// Memorys are memory spaces that are holding actual data. `Memory`s are +/// passed among `Segment`s or passed as input/output of the `Model`. When +/// `size` field is present, `size` specifies the space that is needed. When +/// `size` is not present (or set to -1), the memory is dynamic shaped, and +/// runtime needs to derive size for the `Memory` after input shape has been +/// set. +/// Tensor: +/// Tensor is an abstract description of a chunk of data, with a specific +/// data type (DType) and shape, as well optional strides. Each Tensor is +/// bound to a `Memory` (with memory_id). When present, `offset` is used to +/// describe the relative address of the tensor within the `Memory`. +/// InputTensor/OutputTensor: +/// To describe the input/output tensors of the model, by referencing +/// tensor_id in tensor_list, along with preprocess_hints/postprocess_hints. +/// `preprocess_hints` are information on how input data should be processed +/// before passing to the model, and `postprocess_hints` are information on +/// how output data should be processed. +/// Program: +/// Program is the executive part of a `Segment`, 2 types of Programs are +/// defined, TpuProgram and CpuProgram. +/// TpuProgram: +/// A TpuProgram consists of `Cmdbuf` and `Weight`. To support different +/// batch_size or input image dimesions, multiple `Cmdbuf`s may be needed, +/// with each `Cmdbuf` handling one fixed batch_size and image dims. i.e. +/// on `Cmdbuf` level dynamic shape are not supported. +/// Cmdbuf: +/// Cmdbuf is a sequence of TPU instuctions. +/// Weight: +/// Weight is the trained weight data. +/// CpuProgram: +/// A CpuProgram consists of a cpu function and `Weight`. A function could +/// be either a runtime build-in function, or a plug-in library registered +/// along with the Model in the CpuFunction section. +/// CpuFunction: +/// A CpuFunction is a plug-in library registered along with the Model. +/// + +enum MajorVersion : ubyte { + value = 1 +} +enum MinorVersion : ubyte { + value = 4 +} +enum SubMinorVersion : ubyte { + value = 0 +} + +struct Version { + major_ : ubyte; + minor_ : ubyte; + sub_minor : ubyte; +} + +enum DType : ubyte { + FP32 = 0, + INT32 = 1, + UINT32 = 2, + BF16 = 3, + INT16 = 4, + UINT16 = 5, + INT8 = 6, + UINT8 = 7 +} + +/// -1 is used to indicate that dim is dynamic +table Shape { + dim:[int64] (required); +} + +enum QuantType : ubyte { + NONE = 0, + BF16 = 1, + INT8_SYM = 2, + INT8_ASYM = 3 +} + +/// for symetric quant, only max_value is used (also called threshold) +table QuantInfo { + type:QuantType; + max_value:float; + min_value:float; + zero_point:float; + qscale:float; +} + +table Tensor { + tensor_id:uint32; + name:string (required); + offset:int64; + dtype:DType; + shape:Shape (required); + stride:Shape; + quant:QuantInfo; + overwrote:bool; + scale:[float]; + mean:[float]; + pixel_format:string; + aligned:bool; + size:uint32; +} + +table Weight { + name:string; + offset:int64; + size:uint32; + shape:Shape; + type:DType; +} + +/// +/// color: +/// channel order for input image, valid values are `RGB`, `BGR`. +/// raw_scale: +/// a scale to apply before other proprocessing precedures +/// mean: +/// channel mean value, preprocess will substract the input by this value +/// std: +/// channel std value, preprocess will divide the input by this value +/// input_scale: +/// a scale to apply after other proprocessing precedures +/// +table PreProcessHints { + color:string; + raw_scale:float; + mean:string; + std:string; + input_scale:float; + data_format:string; +} + +table PostProcessHints { + done_softmax:bool; +} + +enum RoutineType: ubyte { + TPU = 0, + CPU = 1 +} + +table TpuRoutine { + cmdbuf_section:string; + dmabuf_section:string; +} + +table CpuRoutine { + function_section:string (required); + function_args:[ubyte] (required); +} + +table Routine { + type:RoutineType; + in_tensors:[string]; + out_tensors:[string]; + tpu_routine:TpuRoutine; + cpu_routine:CpuRoutine; +} + +table Program { + batch_num:uint32; + neuron_size:uint32; + input_tensors:[string] (required); + output_tensors:[string] (required); + tensor_map:[Tensor] (required); + routines:[Routine] (required); + shared_gmem:uint32; + private_gmem:uint32; +} + +enum SectionType: ubyte { + WEIGHT = 0, + CMDBUF = 1, + FUNC_X86 = 2, + FUNC_AARCH64 = 3, + DMABUF = 4 +} + +table Section { + type:SectionType; + name:string (required); + size:uint32; + offset:uint32; + encrypt:bool; + compress:bool; + decompressed_size:uint32; +} + +table Model { + version:Version (required); + name:string (required); + build_time:string; + preprocess_hints:PreProcessHints; + postprocess_hints:PostProcessHints; + weight_map:[Weight]; + programs:[Program] (required); + sections:[Section] (required); + target:string; + mlir_version:string; +} + +root_type Model; diff --git a/cvibuilder/proto/parameter.fbs b/cvibuilder/proto/parameter.fbs new file mode 100644 index 000000000..dae796695 --- /dev/null +++ b/cvibuilder/proto/parameter.fbs @@ -0,0 +1,46 @@ +namespace cvi.cpu_op; + +table IntAttr { + key:string; + value:int; +} + +table FloatAttr { + key:string; + value:float; +} + +table BoolAttr { + key:string; + value:bool; +} + +table StrAttr { + key:string; + value:string; +} + +table IntArrayAttr { + key:string; + value:[int]; +} + +table FloatArrayAttr { + key:string; + value:[float]; +} + +table Attribute { + float_attr:FloatAttr; + bool_attr:BoolAttr; + int_attr:IntAttr; + str_attr:StrAttr; + float_array_attr:FloatArrayAttr; + int_array_attr:IntArrayAttr; +} + +table Parameter { + attributes:[Attribute]; +} + +root_type Parameter; \ No newline at end of file