1775 lines
70 KiB
C++
1775 lines
70 KiB
C++
// Copyright 2020 The Chromium Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
//#define LOG_NDEBUG 0
|
|
#define LOG_TAG "V4L2EncodeComponent"
|
|
|
|
#include <v4l2_codec2/components/V4L2EncodeComponent.h>
|
|
|
|
#include <inttypes.h>
|
|
|
|
#include <algorithm>
|
|
#include <utility>
|
|
|
|
#include <C2AllocatorGralloc.h>
|
|
#include <C2PlatformSupport.h>
|
|
#include <C2Work.h>
|
|
#include <android/hardware/graphics/common/1.0/types.h>
|
|
#include <base/bind.h>
|
|
#include <base/bind_helpers.h>
|
|
#include <log/log.h>
|
|
#include <media/stagefright/MediaDefs.h>
|
|
#include <ui/GraphicBuffer.h>
|
|
|
|
#include <fourcc.h>
|
|
#include <h264_parser.h>
|
|
#include <rect.h>
|
|
#include <v4l2_codec2/common/Common.h>
|
|
#include <v4l2_codec2/common/EncodeHelpers.h>
|
|
#include <v4l2_device.h>
|
|
#include <video_pixel_format.h>
|
|
|
|
using android::hardware::graphics::common::V1_0::BufferUsage;
|
|
|
|
namespace android {
|
|
|
|
namespace {
|
|
|
|
const media::VideoPixelFormat kInputPixelFormat = media::VideoPixelFormat::PIXEL_FORMAT_NV12;
|
|
|
|
// Get the video frame layout from the specified |inputBlock|.
|
|
// TODO(dstaessens): Clean up code extracting layout from a C2GraphicBlock.
|
|
std::optional<std::vector<VideoFramePlane>> getVideoFrameLayout(const C2ConstGraphicBlock& block,
|
|
media::VideoPixelFormat* format) {
|
|
ALOGV("%s()", __func__);
|
|
|
|
// Get the C2PlanarLayout from the graphics block. The C2GraphicView returned by block.map()
|
|
// needs to be released before calling getGraphicBlockInfo(), or the lockYCbCr() call will block
|
|
// Indefinitely.
|
|
C2PlanarLayout layout = block.map().get().layout();
|
|
|
|
// The above layout() cannot fill layout information and memset 0 instead if the input format is
|
|
// IMPLEMENTATION_DEFINED and its backed format is RGB. We fill the layout by using
|
|
// ImplDefinedToRGBXMap in the case.
|
|
if (layout.type == C2PlanarLayout::TYPE_UNKNOWN) {
|
|
std::unique_ptr<ImplDefinedToRGBXMap> idMap = ImplDefinedToRGBXMap::Create(block);
|
|
if (idMap == nullptr) {
|
|
ALOGE("Unable to parse RGBX_8888 from IMPLEMENTATION_DEFINED");
|
|
return std::nullopt;
|
|
}
|
|
layout.type = C2PlanarLayout::TYPE_RGB;
|
|
// These parameters would be used in TYPE_GRB case below.
|
|
layout.numPlanes = 3; // same value as in C2AllocationGralloc::map()
|
|
layout.rootPlanes = 1; // same value as in C2AllocationGralloc::map()
|
|
layout.planes[C2PlanarLayout::PLANE_R].offset = idMap->offset();
|
|
layout.planes[C2PlanarLayout::PLANE_R].rowInc = idMap->rowInc();
|
|
}
|
|
|
|
std::vector<uint32_t> offsets(layout.numPlanes, 0u);
|
|
std::vector<uint32_t> strides(layout.numPlanes, 0u);
|
|
switch (layout.type) {
|
|
case C2PlanarLayout::TYPE_YUV: {
|
|
android_ycbcr ycbcr = getGraphicBlockInfo(block);
|
|
offsets[C2PlanarLayout::PLANE_Y] =
|
|
static_cast<uint32_t>(reinterpret_cast<uintptr_t>(ycbcr.y));
|
|
offsets[C2PlanarLayout::PLANE_U] =
|
|
static_cast<uint32_t>(reinterpret_cast<uintptr_t>(ycbcr.cb));
|
|
offsets[C2PlanarLayout::PLANE_V] =
|
|
static_cast<uint32_t>(reinterpret_cast<uintptr_t>(ycbcr.cr));
|
|
strides[C2PlanarLayout::PLANE_Y] = static_cast<uint32_t>(ycbcr.ystride);
|
|
strides[C2PlanarLayout::PLANE_U] = static_cast<uint32_t>(ycbcr.cstride);
|
|
strides[C2PlanarLayout::PLANE_V] = static_cast<uint32_t>(ycbcr.cstride);
|
|
|
|
bool crcb = false;
|
|
if (offsets[C2PlanarLayout::PLANE_U] > offsets[C2PlanarLayout::PLANE_V]) {
|
|
// Swap offsets, no need to swap strides as they are identical for both chroma planes.
|
|
std::swap(offsets[C2PlanarLayout::PLANE_U], offsets[C2PlanarLayout::PLANE_V]);
|
|
crcb = true;
|
|
}
|
|
|
|
bool semiplanar = false;
|
|
if (ycbcr.chroma_step >
|
|
offsets[C2PlanarLayout::PLANE_V] - offsets[C2PlanarLayout::PLANE_U]) {
|
|
semiplanar = true;
|
|
}
|
|
|
|
if (!crcb && !semiplanar) {
|
|
*format = media::VideoPixelFormat::PIXEL_FORMAT_I420;
|
|
} else if (!crcb && semiplanar) {
|
|
*format = media::VideoPixelFormat::PIXEL_FORMAT_NV12;
|
|
} else if (crcb && !semiplanar) {
|
|
// HACK: pretend YV12 is I420 now since VEA only accepts I420. (YV12 will be used
|
|
// for input byte-buffer mode).
|
|
// TODO(dstaessens): Is this hack still necessary now we're not using the VEA directly?
|
|
//format = media::VideoPixelFormat::PIXEL_FORMAT_YV12;
|
|
*format = media::VideoPixelFormat::PIXEL_FORMAT_I420;
|
|
} else {
|
|
*format = media::VideoPixelFormat::PIXEL_FORMAT_NV21;
|
|
}
|
|
break;
|
|
}
|
|
case C2PlanarLayout::TYPE_RGB: {
|
|
offsets[C2PlanarLayout::PLANE_R] = layout.planes[C2PlanarLayout::PLANE_R].offset;
|
|
strides[C2PlanarLayout::PLANE_R] =
|
|
static_cast<uint32_t>(layout.planes[C2PlanarLayout::PLANE_R].rowInc);
|
|
*format = media::VideoPixelFormat::PIXEL_FORMAT_ARGB;
|
|
break;
|
|
}
|
|
default:
|
|
ALOGW("Unknown layout type: %u", static_cast<uint32_t>(layout.type));
|
|
return std::nullopt;
|
|
}
|
|
|
|
std::vector<VideoFramePlane> planes;
|
|
for (uint32_t i = 0; i < layout.rootPlanes; ++i) {
|
|
planes.push_back({offsets[i], strides[i]});
|
|
}
|
|
return planes;
|
|
}
|
|
|
|
// The maximum size for output buffer, which is chosen empirically for a 1080p video.
|
|
constexpr size_t kMaxBitstreamBufferSizeInBytes = 2 * 1024 * 1024; // 2MB
|
|
// The frame size for 1080p (FHD) video in pixels.
|
|
constexpr int k1080PSizeInPixels = 1920 * 1080;
|
|
// The frame size for 1440p (QHD) video in pixels.
|
|
constexpr int k1440PSizeInPixels = 2560 * 1440;
|
|
|
|
// Use quadruple size of kMaxBitstreamBufferSizeInBytes when the input frame size is larger than
|
|
// 1440p, double if larger than 1080p. This is chosen empirically for some 4k encoding use cases and
|
|
// the Android CTS VideoEncoderTest (crbug.com/927284).
|
|
size_t GetMaxOutputBufferSize(const media::Size& size) {
|
|
if (size.GetArea() > k1440PSizeInPixels) return kMaxBitstreamBufferSizeInBytes * 4;
|
|
if (size.GetArea() > k1080PSizeInPixels) return kMaxBitstreamBufferSizeInBytes * 2;
|
|
return kMaxBitstreamBufferSizeInBytes;
|
|
}
|
|
|
|
// These are rather subjectively tuned.
|
|
constexpr size_t kInputBufferCount = 2;
|
|
constexpr size_t kOutputBufferCount = 2;
|
|
|
|
// Define V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR control code if not present in header files.
|
|
#ifndef V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR
|
|
#define V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR (V4L2_CID_MPEG_BASE + 388)
|
|
#endif
|
|
|
|
} // namespace
|
|
|
|
// static
|
|
std::unique_ptr<V4L2EncodeComponent::InputFrame> V4L2EncodeComponent::InputFrame::Create(
|
|
const C2ConstGraphicBlock& block) {
|
|
std::vector<int> fds;
|
|
const C2Handle* const handle = block.handle();
|
|
for (int i = 0; i < handle->numFds; i++) {
|
|
fds.emplace_back(handle->data[i]);
|
|
}
|
|
|
|
return std::unique_ptr<InputFrame>(new InputFrame(std::move(fds)));
|
|
}
|
|
|
|
// static
|
|
std::shared_ptr<C2Component> V4L2EncodeComponent::create(
|
|
C2String name, c2_node_id_t id, std::shared_ptr<C2ReflectorHelper> helper,
|
|
C2ComponentFactory::ComponentDeleter deleter) {
|
|
ALOGV("%s(%s)", __func__, name.c_str());
|
|
|
|
auto interface = std::make_shared<V4L2EncodeInterface>(name, std::move(helper));
|
|
if (interface->status() != C2_OK) {
|
|
ALOGE("Component interface initialization failed (error code %d)", interface->status());
|
|
return nullptr;
|
|
}
|
|
|
|
return std::shared_ptr<C2Component>(new V4L2EncodeComponent(name, id, std::move(interface)),
|
|
deleter);
|
|
}
|
|
|
|
V4L2EncodeComponent::V4L2EncodeComponent(C2String name, c2_node_id_t id,
|
|
std::shared_ptr<V4L2EncodeInterface> interface)
|
|
: mName(name),
|
|
mId(id),
|
|
mInterface(std::move(interface)),
|
|
mComponentState(ComponentState::LOADED) {
|
|
ALOGV("%s(%s)", __func__, name.c_str());
|
|
}
|
|
|
|
V4L2EncodeComponent::~V4L2EncodeComponent() {
|
|
ALOGV("%s()", __func__);
|
|
|
|
// Stop encoder thread and invalidate pointers if component wasn't stopped before destroying.
|
|
if (mEncoderThread.IsRunning()) {
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE, ::base::BindOnce(
|
|
[](::base::WeakPtrFactory<V4L2EncodeComponent>* weakPtrFactory) {
|
|
weakPtrFactory->InvalidateWeakPtrs();
|
|
},
|
|
&mWeakThisFactory));
|
|
mEncoderThread.Stop();
|
|
}
|
|
ALOGV("%s(): done", __func__);
|
|
}
|
|
|
|
c2_status_t V4L2EncodeComponent::start() {
|
|
ALOGV("%s()", __func__);
|
|
|
|
// Lock while starting, to synchronize start/stop/reset/release calls.
|
|
std::lock_guard<std::mutex> lock(mComponentLock);
|
|
|
|
// According to the specification start() should only be called in the LOADED state.
|
|
if (mComponentState != ComponentState::LOADED) {
|
|
return C2_BAD_STATE;
|
|
}
|
|
|
|
if (!mEncoderThread.Start()) {
|
|
ALOGE("Failed to start encoder thread");
|
|
return C2_CORRUPTED;
|
|
}
|
|
mEncoderTaskRunner = mEncoderThread.task_runner();
|
|
mWeakThis = mWeakThisFactory.GetWeakPtr();
|
|
|
|
// Initialize the encoder on the encoder thread.
|
|
::base::WaitableEvent done;
|
|
bool success = false;
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE, ::base::Bind(&V4L2EncodeComponent::startTask, mWeakThis, &success, &done));
|
|
done.Wait();
|
|
|
|
if (!success) {
|
|
ALOGE("Failed to initialize encoder");
|
|
return C2_CORRUPTED;
|
|
}
|
|
|
|
setComponentState(ComponentState::RUNNING);
|
|
return C2_OK;
|
|
}
|
|
|
|
c2_status_t V4L2EncodeComponent::stop() {
|
|
ALOGV("%s()", __func__);
|
|
|
|
// Lock while stopping, to synchronize start/stop/reset/release calls.
|
|
std::lock_guard<std::mutex> lock(mComponentLock);
|
|
|
|
if (mComponentState != ComponentState::RUNNING && mComponentState != ComponentState::ERROR) {
|
|
return C2_BAD_STATE;
|
|
}
|
|
|
|
// Return immediately if the component is already stopped.
|
|
if (!mEncoderThread.IsRunning()) {
|
|
return C2_OK;
|
|
}
|
|
|
|
// Wait for the component to stop.
|
|
::base::WaitableEvent done;
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE, ::base::BindOnce(&V4L2EncodeComponent::stopTask, mWeakThis, &done));
|
|
done.Wait();
|
|
mEncoderThread.Stop();
|
|
|
|
setComponentState(ComponentState::LOADED);
|
|
|
|
ALOGV("%s() - done", __func__);
|
|
return C2_OK;
|
|
}
|
|
|
|
c2_status_t V4L2EncodeComponent::reset() {
|
|
ALOGV("%s()", __func__);
|
|
|
|
// The interface specification says: "This method MUST be supported in all (including tripped)
|
|
// states other than released".
|
|
if (mComponentState == ComponentState::UNLOADED) {
|
|
return C2_BAD_STATE;
|
|
}
|
|
|
|
// TODO(dstaessens): Reset the component's interface to default values.
|
|
stop();
|
|
|
|
return C2_OK;
|
|
}
|
|
|
|
c2_status_t V4L2EncodeComponent::release() {
|
|
ALOGV("%s()", __func__);
|
|
|
|
// The interface specification says: "This method MUST be supported in stopped state.", but the
|
|
// release method seems to be called in other states as well.
|
|
reset();
|
|
|
|
setComponentState(ComponentState::UNLOADED);
|
|
return C2_OK;
|
|
}
|
|
|
|
c2_status_t V4L2EncodeComponent::queue_nb(std::list<std::unique_ptr<C2Work>>* const items) {
|
|
ALOGV("%s()", __func__);
|
|
|
|
if (mComponentState != ComponentState::RUNNING) {
|
|
ALOGE("Trying to queue work item while component is not running");
|
|
return C2_BAD_STATE;
|
|
}
|
|
|
|
while (!items->empty()) {
|
|
mEncoderTaskRunner->PostTask(FROM_HERE,
|
|
::base::BindOnce(&V4L2EncodeComponent::queueTask, mWeakThis,
|
|
std::move(items->front())));
|
|
items->pop_front();
|
|
}
|
|
|
|
return C2_OK;
|
|
}
|
|
|
|
c2_status_t V4L2EncodeComponent::drain_nb(drain_mode_t mode) {
|
|
ALOGV("%s()", __func__);
|
|
|
|
if (mode == DRAIN_CHAIN) {
|
|
return C2_OMITTED; // Tunneling is not supported for now.
|
|
}
|
|
|
|
if (mComponentState != ComponentState::RUNNING) {
|
|
return C2_BAD_STATE;
|
|
}
|
|
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE, ::base::BindOnce(&V4L2EncodeComponent::drainTask, mWeakThis, mode));
|
|
return C2_OK;
|
|
}
|
|
|
|
c2_status_t V4L2EncodeComponent::flush_sm(flush_mode_t mode,
|
|
std::list<std::unique_ptr<C2Work>>* const flushedWork) {
|
|
ALOGV("%s()", __func__);
|
|
|
|
if (mode != FLUSH_COMPONENT) {
|
|
return C2_OMITTED; // Tunneling is not supported by now
|
|
}
|
|
|
|
if (mComponentState != ComponentState::RUNNING) {
|
|
return C2_BAD_STATE;
|
|
}
|
|
|
|
// Work that can be immediately discarded should be returned in |flushedWork|. This method may
|
|
// be momentarily blocking but must return within 5ms, which should give us enough time to
|
|
// immediately abandon all non-started work on the encoder thread. We can return all work that
|
|
// can't be immediately discarded using onWorkDone() later.
|
|
::base::WaitableEvent done;
|
|
mEncoderTaskRunner->PostTask(FROM_HERE, ::base::BindOnce(&V4L2EncodeComponent::flushTask,
|
|
mWeakThis, &done, flushedWork));
|
|
done.Wait();
|
|
|
|
return C2_OK;
|
|
}
|
|
|
|
c2_status_t V4L2EncodeComponent::announce_nb(const std::vector<C2WorkOutline>& items) {
|
|
return C2_OMITTED; // Tunneling is not supported by now
|
|
}
|
|
|
|
c2_status_t V4L2EncodeComponent::setListener_vb(const std::shared_ptr<Listener>& listener,
|
|
c2_blocking_t mayBlock) {
|
|
ALOG_ASSERT(mComponentState != ComponentState::UNLOADED);
|
|
|
|
// Lock so we're sure the component isn't currently starting or stopping.
|
|
std::lock_guard<std::mutex> lock(mComponentLock);
|
|
|
|
// If the encoder thread is not running it's safe to update the listener directly.
|
|
if (!mEncoderThread.IsRunning()) {
|
|
mListener = listener;
|
|
return C2_OK;
|
|
}
|
|
|
|
// The listener should be updated before exiting this function. If called while the component is
|
|
// currently running we should be allowed to block, as we can only change the listener on the
|
|
// encoder thread.
|
|
ALOG_ASSERT(mayBlock == c2_blocking_t::C2_MAY_BLOCK);
|
|
|
|
::base::WaitableEvent done;
|
|
mEncoderTaskRunner->PostTask(FROM_HERE, ::base::BindOnce(&V4L2EncodeComponent::setListenerTask,
|
|
mWeakThis, listener, &done));
|
|
done.Wait();
|
|
|
|
return C2_OK;
|
|
}
|
|
|
|
std::shared_ptr<C2ComponentInterface> V4L2EncodeComponent::intf() {
|
|
return std::make_shared<SimpleInterface<V4L2EncodeInterface>>(mName.c_str(), mId, mInterface);
|
|
}
|
|
|
|
void V4L2EncodeComponent::startTask(bool* success, ::base::WaitableEvent* done) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED);
|
|
|
|
*success = initializeEncoder();
|
|
done->Signal();
|
|
}
|
|
|
|
void V4L2EncodeComponent::stopTask(::base::WaitableEvent* done) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
// Flushing the encoder will abort all pending work and stop polling and streaming on the V4L2
|
|
// device queues.
|
|
flush();
|
|
|
|
// Deallocate all V4L2 device input and output buffers.
|
|
destroyInputBuffers();
|
|
destroyOutputBuffers();
|
|
|
|
// Invalidate all weak pointers so no more functions will be executed on the encoder thread.
|
|
mWeakThisFactory.InvalidateWeakPtrs();
|
|
|
|
setEncoderState(EncoderState::UNINITIALIZED);
|
|
done->Signal();
|
|
}
|
|
|
|
void V4L2EncodeComponent::queueTask(std::unique_ptr<C2Work> work) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED);
|
|
|
|
// If we're in the error state we can immediately return, freeing all buffers in the work item.
|
|
if (mEncoderState == EncoderState::ERROR) {
|
|
return;
|
|
}
|
|
|
|
ALOGV("Queued work item (index: %llu, timestamp: %llu, EOS: %d)",
|
|
work->input.ordinal.frameIndex.peekull(), work->input.ordinal.timestamp.peekull(),
|
|
work->input.flags & C2FrameData::FLAG_END_OF_STREAM);
|
|
|
|
mInputWorkQueue.push(std::move(work));
|
|
|
|
// If we were waiting for work, start encoding again.
|
|
if (mEncoderState == EncoderState::WAITING_FOR_INPUT) {
|
|
setEncoderState(EncoderState::ENCODING);
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE,
|
|
::base::BindOnce(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis));
|
|
}
|
|
}
|
|
|
|
// TODO(dstaessens): Investigate improving drain logic after draining the virtio device is fixed.
|
|
void V4L2EncodeComponent::drainTask(drain_mode_t /*drainMode*/) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
// We can only start draining if all the work in our input queue has been queued on the V4L2
|
|
// device input queue, so we mark the last item in the input queue as EOS.
|
|
if (!mInputWorkQueue.empty()) {
|
|
ALOGV("Marking last item in input work queue as EOS");
|
|
mInputWorkQueue.back()->input.flags = static_cast<C2FrameData::flags_t>(
|
|
mInputWorkQueue.back()->input.flags | C2FrameData::FLAG_END_OF_STREAM);
|
|
return;
|
|
}
|
|
|
|
// If the input queue is empty and there is only a single empty EOS work item in the output
|
|
// queue we can immediately consider flushing done.
|
|
if ((mOutputWorkQueue.size() == 1) && mOutputWorkQueue.back()->input.buffers.empty()) {
|
|
ALOG_ASSERT(mOutputWorkQueue.back()->input.flags & C2FrameData::FLAG_END_OF_STREAM);
|
|
setEncoderState(EncoderState::DRAINING);
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE, ::base::BindOnce(&V4L2EncodeComponent::onDrainDone, mWeakThis, true));
|
|
return;
|
|
}
|
|
|
|
// If the input queue is empty all work that needs to be drained has already been queued in the
|
|
// V4L2 device, so we can immediately request a drain.
|
|
if (!mOutputWorkQueue.empty()) {
|
|
// Mark the last item in the output work queue as EOS, so we will only report it as
|
|
// finished after draining has completed.
|
|
ALOGV("Starting drain and marking last item in output work queue as EOS");
|
|
mOutputWorkQueue.back()->input.flags = C2FrameData::FLAG_END_OF_STREAM;
|
|
drain();
|
|
}
|
|
}
|
|
|
|
void V4L2EncodeComponent::onDrainDone(bool done) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState == EncoderState::DRAINING || mEncoderState == EncoderState::ERROR);
|
|
|
|
if (mEncoderState == EncoderState::ERROR) {
|
|
return;
|
|
}
|
|
|
|
if (!done) {
|
|
ALOGE("draining the encoder failed");
|
|
reportError(C2_CORRUPTED);
|
|
return;
|
|
}
|
|
|
|
// The last work item in the output work queue should be an EOS request.
|
|
if (mOutputWorkQueue.empty() ||
|
|
!(mOutputWorkQueue.back()->input.flags & C2FrameData::FLAG_END_OF_STREAM)) {
|
|
ALOGE("The last item in the output work queue should be marked EOS");
|
|
reportError(C2_CORRUPTED);
|
|
return;
|
|
}
|
|
|
|
// Mark the last item in the output work queue as EOS done.
|
|
C2Work* eosWork = mOutputWorkQueue.back().get();
|
|
eosWork->worklets.back()->output.flags = C2FrameData::FLAG_END_OF_STREAM;
|
|
|
|
// Draining is done which means all buffers on the device output queue have been returned, but
|
|
// not all buffers on the device input queue might have been returned yet.
|
|
if ((mOutputWorkQueue.size() > 1) || !isWorkDone(*eosWork)) {
|
|
ALOGV("Draining done, waiting for input buffers to be returned");
|
|
return;
|
|
}
|
|
|
|
ALOGV("Draining done");
|
|
reportWork(std::move(mOutputWorkQueue.front()));
|
|
mOutputWorkQueue.pop_front();
|
|
|
|
// Draining the encoder is now done, we can start encoding again.
|
|
if (!mInputWorkQueue.empty()) {
|
|
setEncoderState(EncoderState::ENCODING);
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE,
|
|
::base::BindOnce(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis));
|
|
} else {
|
|
setEncoderState(EncoderState::WAITING_FOR_INPUT);
|
|
}
|
|
}
|
|
|
|
void V4L2EncodeComponent::flushTask(::base::WaitableEvent* done,
|
|
std::list<std::unique_ptr<C2Work>>* const flushedWork) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
// Move all work that can immediately be aborted to flushedWork, and notify the caller.
|
|
if (flushedWork) {
|
|
while (!mInputWorkQueue.empty()) {
|
|
std::unique_ptr<C2Work> work = std::move(mInputWorkQueue.front());
|
|
work->input.buffers.clear();
|
|
flushedWork->push_back(std::move(work));
|
|
mInputWorkQueue.pop();
|
|
}
|
|
}
|
|
done->Signal();
|
|
|
|
flush();
|
|
}
|
|
|
|
void V4L2EncodeComponent::setListenerTask(const std::shared_ptr<Listener>& listener,
|
|
::base::WaitableEvent* done) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
mListener = listener;
|
|
done->Signal();
|
|
}
|
|
|
|
bool V4L2EncodeComponent::initializeEncoder() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED);
|
|
|
|
mVisibleSize = mInterface->getInputVisibleSize();
|
|
mKeyFramePeriod = mInterface->getKeyFramePeriod();
|
|
mKeyFrameCounter = 0;
|
|
mCSDSubmitted = false;
|
|
|
|
// Open the V4L2 device for encoding to the requested output format.
|
|
// TODO(dstaessens): Do we need to close the device first if already opened?
|
|
// TODO(dstaessens): Avoid conversion to VideoCodecProfile and use C2Config::profile_t directly.
|
|
media::VideoCodecProfile outputProfile =
|
|
c2ProfileToVideoCodecProfile(mInterface->getOutputProfile());
|
|
uint32_t outputPixelFormat =
|
|
media::V4L2Device::VideoCodecProfileToV4L2PixFmt(outputProfile, false);
|
|
if (!outputPixelFormat) {
|
|
ALOGE("Invalid output profile %s", media::GetProfileName(outputProfile).c_str());
|
|
return false;
|
|
}
|
|
|
|
mDevice = media::V4L2Device::Create();
|
|
if (!mDevice) {
|
|
ALOGE("Failed to create V4L2 device");
|
|
return false;
|
|
}
|
|
|
|
if (!mDevice->Open(media::V4L2Device::Type::kEncoder, outputPixelFormat)) {
|
|
ALOGE("Failed to open device for profile %s (%s)",
|
|
media::GetProfileName(outputProfile).c_str(),
|
|
media::FourccToString(outputPixelFormat).c_str());
|
|
return false;
|
|
}
|
|
|
|
// Make sure the device has all required capabilities (multi-planar Memory-To-Memory and
|
|
// streaming I/O), and whether flushing is supported.
|
|
if (!mDevice->HasCapabilities(V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING)) {
|
|
ALOGE("Device doesn't have the required capabilities");
|
|
return false;
|
|
}
|
|
if (!mDevice->IsCommandSupported(V4L2_ENC_CMD_STOP)) {
|
|
ALOGE("Device does not support flushing (V4L2_ENC_CMD_STOP)");
|
|
return false;
|
|
}
|
|
|
|
// Get input/output queues so we can send encode request to the device and get back the results.
|
|
mInputQueue = mDevice->GetQueue(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
|
|
mOutputQueue = mDevice->GetQueue(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
|
|
if (!mInputQueue || !mOutputQueue) {
|
|
ALOGE("Failed to get V4L2 device queues");
|
|
return false;
|
|
}
|
|
|
|
// First try to configure the specified output format, as changing the output format can affect
|
|
// the configured input format.
|
|
if (!configureOutputFormat(outputProfile)) return false;
|
|
|
|
// Configure the input format. If the device doesn't support the specified format we'll use one
|
|
// of the device's preferred formats in combination with an input format convertor.
|
|
if (!configureInputFormat(kInputPixelFormat)) return false;
|
|
|
|
// Create input and output buffers.
|
|
// TODO(dstaessens): Avoid allocating output buffers, encode directly into blockpool buffers.
|
|
if (!createInputBuffers() || !createOutputBuffers()) return false;
|
|
|
|
// Configure the device, setting all required controls.
|
|
uint8_t level = c2LevelToLevelIDC(mInterface->getOutputLevel());
|
|
if (!configureDevice(outputProfile, level)) return false;
|
|
|
|
// We're ready to start encoding now.
|
|
setEncoderState(EncoderState::WAITING_FOR_INPUT);
|
|
|
|
// As initialization is asynchronous work might have already be queued.
|
|
if (!mInputWorkQueue.empty()) {
|
|
setEncoderState(EncoderState::ENCODING);
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE, ::base::Bind(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::configureInputFormat(media::VideoPixelFormat inputFormat) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED);
|
|
ALOG_ASSERT(!mInputQueue->IsStreaming());
|
|
ALOG_ASSERT(!mVisibleSize.IsEmpty());
|
|
ALOG_ASSERT(!mInputFormatConverter);
|
|
|
|
// First try to use the requested pixel format directly.
|
|
::base::Optional<struct v4l2_format> format;
|
|
auto fourcc = media::Fourcc::FromVideoPixelFormat(inputFormat, false);
|
|
if (fourcc) {
|
|
format = mInputQueue->SetFormat(fourcc->ToV4L2PixFmt(), mVisibleSize, 0);
|
|
}
|
|
|
|
// If the device doesn't support the requested input format we'll try the device's preferred
|
|
// input pixel formats and use a format convertor. We need to try all formats as some formats
|
|
// might not be supported for the configured output format.
|
|
if (!format) {
|
|
std::vector<uint32_t> preferredFormats =
|
|
mDevice->PreferredInputFormat(media::V4L2Device::Type::kEncoder);
|
|
for (uint32_t i = 0; !format && i < preferredFormats.size(); ++i) {
|
|
format = mInputQueue->SetFormat(preferredFormats[i], mVisibleSize, 0);
|
|
}
|
|
}
|
|
|
|
if (!format) {
|
|
ALOGE("Failed to set input format to %s",
|
|
media::VideoPixelFormatToString(inputFormat).c_str());
|
|
return false;
|
|
}
|
|
|
|
// Check whether the negotiated input format is valid. The coded size might be adjusted to match
|
|
// encoder minimums, maximums and alignment requirements of the currently selected formats.
|
|
auto layout = media::V4L2Device::V4L2FormatToVideoFrameLayout(*format);
|
|
if (!layout) {
|
|
ALOGE("Invalid input layout");
|
|
return false;
|
|
}
|
|
|
|
mInputLayout = layout.value();
|
|
if (!media::Rect(mInputLayout->coded_size()).Contains(media::Rect(mVisibleSize))) {
|
|
ALOGE("Input size %s exceeds encoder capability, encoder can handle %s",
|
|
mVisibleSize.ToString().c_str(), mInputLayout->coded_size().ToString().c_str());
|
|
return false;
|
|
}
|
|
|
|
// Calculate the input coded size from the format.
|
|
// TODO(dstaessens): How is this different from mInputLayout->coded_size()?
|
|
mInputCodedSize = media::V4L2Device::AllocatedSizeFromV4L2Format(*format);
|
|
|
|
// Add an input format convertor if the device doesn't support the requested input format.
|
|
// Note: The amount of input buffers in the convertor should match the amount of buffers on the
|
|
// device input queue, to simplify logic.
|
|
// TODO(dstaessens): Currently an input format convertor is always required. Mapping an input
|
|
// buffer always seems to fail unless we copy it into a new a buffer first. As a temporary
|
|
// workaround the line below is commented, but this should be undone once the issue is fixed.
|
|
//if (mInputLayout->format() != inputFormat) {
|
|
ALOGV("Creating input format convertor (%s)",
|
|
media::VideoPixelFormatToString(mInputLayout->format()).c_str());
|
|
mInputFormatConverter =
|
|
FormatConverter::Create(inputFormat, mVisibleSize, kInputBufferCount, mInputCodedSize);
|
|
if (!mInputFormatConverter) {
|
|
ALOGE("Failed to created input format convertor");
|
|
return false;
|
|
}
|
|
//}
|
|
|
|
// The coded input size might be different from the visible size due to alignment requirements,
|
|
// So we need to specify the visible rectangle. Note that this rectangle might still be adjusted
|
|
// due to hardware limitations.
|
|
// TODO(dstaessens): Overwrite mVisibleSize with the adapted visible size here?
|
|
media::Rect visibleRectangle(mVisibleSize.width(), mVisibleSize.height());
|
|
|
|
struct v4l2_rect rect;
|
|
rect.left = visibleRectangle.x();
|
|
rect.top = visibleRectangle.y();
|
|
rect.width = visibleRectangle.width();
|
|
rect.height = visibleRectangle.height();
|
|
|
|
// Try to adjust the visible rectangle using the VIDIOC_S_SELECTION command. If this is not
|
|
// supported we'll try to use the VIDIOC_S_CROP command instead. The visible rectangle might be
|
|
// adjusted to conform to hardware limitations (e.g. round to closest horizontal and vertical
|
|
// offsets, width and height).
|
|
struct v4l2_selection selection_arg;
|
|
memset(&selection_arg, 0, sizeof(selection_arg));
|
|
selection_arg.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
|
|
selection_arg.target = V4L2_SEL_TGT_CROP;
|
|
selection_arg.r = rect;
|
|
if (mDevice->Ioctl(VIDIOC_S_SELECTION, &selection_arg) == 0) {
|
|
visibleRectangle = media::Rect(selection_arg.r.left, selection_arg.r.top,
|
|
selection_arg.r.width, selection_arg.r.height);
|
|
} else {
|
|
struct v4l2_crop crop;
|
|
memset(&crop, 0, sizeof(v4l2_crop));
|
|
crop.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
|
|
crop.c = rect;
|
|
if (mDevice->Ioctl(VIDIOC_S_CROP, &crop) != 0 ||
|
|
mDevice->Ioctl(VIDIOC_G_CROP, &crop) != 0) {
|
|
ALOGE("Failed to crop to specified visible rectangle");
|
|
return false;
|
|
}
|
|
visibleRectangle = media::Rect(crop.c.left, crop.c.top, crop.c.width, crop.c.height);
|
|
}
|
|
|
|
ALOGV("Input format set to %s (size: %s, adjusted size: %dx%d, coded size: %s)",
|
|
media::VideoPixelFormatToString(mInputLayout->format()).c_str(),
|
|
mVisibleSize.ToString().c_str(), visibleRectangle.width(), visibleRectangle.height(),
|
|
mInputCodedSize.ToString().c_str());
|
|
|
|
mVisibleSize.SetSize(visibleRectangle.width(), visibleRectangle.height());
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::configureOutputFormat(media::VideoCodecProfile outputProfile) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED);
|
|
ALOG_ASSERT(!mOutputQueue->IsStreaming());
|
|
ALOG_ASSERT(!mVisibleSize.IsEmpty());
|
|
|
|
auto format = mOutputQueue->SetFormat(
|
|
media::V4L2Device::VideoCodecProfileToV4L2PixFmt(outputProfile, false), mVisibleSize,
|
|
GetMaxOutputBufferSize(mVisibleSize));
|
|
if (!format) {
|
|
ALOGE("Failed to set output format to %s", media::GetProfileName(outputProfile).c_str());
|
|
return false;
|
|
}
|
|
|
|
// The device might adjust the requested output buffer size to match hardware requirements.
|
|
mOutputBufferSize = ::base::checked_cast<size_t>(format->fmt.pix_mp.plane_fmt[0].sizeimage);
|
|
|
|
ALOGV("Output format set to %s (buffer size: %u)", media::GetProfileName(outputProfile).c_str(),
|
|
mOutputBufferSize);
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::configureDevice(media::VideoCodecProfile outputProfile,
|
|
std::optional<const uint8_t> outputH264Level) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
// Enable frame-level bitrate control. This is the only mandatory general control.
|
|
if (!mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG,
|
|
{media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE, 1)})) {
|
|
ALOGW("Failed enabling bitrate control");
|
|
// TODO(b/161508368): V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE is currently not supported yet,
|
|
// assume the operation was successful for now.
|
|
}
|
|
|
|
// Additional optional controls:
|
|
// - Enable macroblock-level bitrate control.
|
|
// - Set GOP length to 0 to disable periodic key frames.
|
|
mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG,
|
|
{media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE, 1),
|
|
media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0)});
|
|
|
|
// All controls below are H.264-specific, so we can return here if the profile is not H.264.
|
|
if (outputProfile < media::H264PROFILE_MIN || outputProfile > media::H264PROFILE_MAX) {
|
|
return true;
|
|
}
|
|
|
|
// When encoding H.264 we want to prepend SPS and PPS to each IDR for resilience. Some
|
|
// devices support this through the V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR control.
|
|
// TODO(b/161495502): V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR is currently not supported
|
|
// yet, just log a warning if the operation was unsuccessful for now.
|
|
if (mDevice->IsCtrlExposed(V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR)) {
|
|
if (!mDevice->SetExtCtrls(
|
|
V4L2_CTRL_CLASS_MPEG,
|
|
{media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_H264_SPS_PPS_BEFORE_IDR, 1)})) {
|
|
ALOGE("Failed to configure device to prepend SPS and PPS to each IDR");
|
|
return false;
|
|
}
|
|
ALOGV("Device supports prepending SPS and PPS to each IDR");
|
|
} else {
|
|
ALOGW("Device doesn't support prepending SPS and PPS to IDR");
|
|
}
|
|
|
|
std::vector<media::V4L2ExtCtrl> h264Ctrls;
|
|
|
|
// No B-frames, for lowest decoding latency.
|
|
h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_B_FRAMES, 0);
|
|
// Quantization parameter maximum value (for variable bitrate control).
|
|
h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_H264_MAX_QP, 51);
|
|
|
|
// Set H.264 profile.
|
|
int32_t profile = media::V4L2Device::VideoCodecProfileToV4L2H264Profile(outputProfile);
|
|
if (profile < 0) {
|
|
ALOGE("Trying to set invalid H.264 profile");
|
|
return false;
|
|
}
|
|
h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_H264_PROFILE, profile);
|
|
|
|
// Set H.264 output level. Use Level 4.0 as fallback default.
|
|
// TODO(dstaessens): Investigate code added by hiroh@ recently to select level in Chrome VEA.
|
|
uint8_t h264Level = outputH264Level.value_or(media::H264SPS::kLevelIDC4p0);
|
|
h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_H264_LEVEL,
|
|
media::V4L2Device::H264LevelIdcToV4L2H264Level(h264Level));
|
|
|
|
// Ask not to put SPS and PPS into separate bitstream buffers.
|
|
h264Ctrls.emplace_back(V4L2_CID_MPEG_VIDEO_HEADER_MODE,
|
|
V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME);
|
|
|
|
// Ignore return value as these controls are optional.
|
|
mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG, std::move(h264Ctrls));
|
|
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::updateEncodingParameters() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
// Query the interface for the encoding parameters requested by the codec 2.0 framework.
|
|
C2StreamBitrateInfo::output bitrateInfo;
|
|
C2StreamFrameRateInfo::output framerateInfo;
|
|
c2_status_t status =
|
|
mInterface->query({&bitrateInfo, &framerateInfo}, {}, C2_DONT_BLOCK, nullptr);
|
|
if (status != C2_OK) {
|
|
ALOGE("Failed to query interface for encoding parameters (error code: %d)", status);
|
|
reportError(status);
|
|
return false;
|
|
}
|
|
|
|
// Ask device to change bitrate if it's different from the currently configured bitrate.
|
|
uint32_t bitrate = bitrateInfo.value;
|
|
if (mBitrate != bitrate) {
|
|
ALOG_ASSERT(bitrate > 0u);
|
|
ALOGV("Setting bitrate to %u", bitrate);
|
|
if (!mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG,
|
|
{media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_BITRATE, bitrate)})) {
|
|
// TODO(b/161495749): V4L2_CID_MPEG_VIDEO_BITRATE is currently not supported yet, assume
|
|
// the operation was successful for now.
|
|
ALOGW("Requesting bitrate change failed");
|
|
}
|
|
mBitrate = bitrate;
|
|
}
|
|
|
|
// Ask device to change framerate if it's different from the currently configured framerate.
|
|
// TODO(dstaessens): Move IOCTL to device and use helper function.
|
|
uint32_t framerate = static_cast<uint32_t>(std::round(framerateInfo.value));
|
|
if (mFramerate != framerate) {
|
|
ALOG_ASSERT(framerate > 0u);
|
|
ALOGV("Setting framerate to %u", framerate);
|
|
struct v4l2_streamparm parms;
|
|
memset(&parms, 0, sizeof(v4l2_streamparm));
|
|
parms.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
|
|
parms.parm.output.timeperframe.numerator = 1;
|
|
parms.parm.output.timeperframe.denominator = framerate;
|
|
if (mDevice->Ioctl(VIDIOC_S_PARM, &parms) != 0) {
|
|
// TODO(b/161499573): VIDIOC_S_PARM is currently not supported yet, assume the operation
|
|
// was successful for now.
|
|
ALOGW("Requesting framerate change failed");
|
|
}
|
|
mFramerate = framerate;
|
|
}
|
|
|
|
// Check whether an explicit key frame was requested, if so reset the key frame counter to
|
|
// immediately request a key frame.
|
|
C2StreamRequestSyncFrameTuning::output requestKeyFrame;
|
|
status = mInterface->query({&requestKeyFrame}, {}, C2_DONT_BLOCK, nullptr);
|
|
if (status != C2_OK) {
|
|
ALOGE("Failed to query interface for key frame request (error code: %d)", status);
|
|
reportError(status);
|
|
return false;
|
|
}
|
|
if (requestKeyFrame.value == C2_TRUE) {
|
|
mKeyFrameCounter = 0;
|
|
requestKeyFrame.value = C2_FALSE;
|
|
std::vector<std::unique_ptr<C2SettingResult>> failures;
|
|
status = mInterface->config({&requestKeyFrame}, C2_MAY_BLOCK, &failures);
|
|
if (status != C2_OK) {
|
|
ALOGE("Failed to reset key frame request on interface (error code: %d)", status);
|
|
reportError(status);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Request the next frame to be a key frame each time the counter reaches 0.
|
|
if (mKeyFrameCounter == 0) {
|
|
if (!mDevice->SetExtCtrls(V4L2_CTRL_CLASS_MPEG,
|
|
{media::V4L2ExtCtrl(V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME)})) {
|
|
// TODO(b/161498590): V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME is currently not supported
|
|
// yet, assume the operation was successful for now.
|
|
ALOGW("Failed requesting key frame");
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void V4L2EncodeComponent::scheduleNextEncodeTask() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState == EncoderState::ENCODING || mEncoderState == EncoderState::ERROR);
|
|
|
|
// If we're in the error state we can immediately return.
|
|
if (mEncoderState == EncoderState::ERROR) {
|
|
return;
|
|
}
|
|
|
|
// Get the next work item. Currently only a single worklet per work item is supported. An input
|
|
// buffer should always be supplied unless this is a drain or CSD request.
|
|
ALOG_ASSERT(!mInputWorkQueue.empty());
|
|
C2Work* work = mInputWorkQueue.front().get();
|
|
ALOG_ASSERT(work->input.buffers.size() <= 1u && work->worklets.size() == 1u);
|
|
|
|
// Set the default values for the output worklet.
|
|
work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0);
|
|
work->worklets.front()->output.buffers.clear();
|
|
work->worklets.front()->output.ordinal = work->input.ordinal;
|
|
|
|
uint64_t index = work->input.ordinal.frameIndex.peeku();
|
|
int64_t timestamp = static_cast<int64_t>(work->input.ordinal.timestamp.peeku());
|
|
bool endOfStream = work->input.flags & C2FrameData::FLAG_END_OF_STREAM;
|
|
ALOGV("Scheduling next encode (index: %" PRIu64 ", timestamp: %" PRId64 ", EOS: %d)", index,
|
|
timestamp, endOfStream);
|
|
|
|
if (!work->input.buffers.empty()) {
|
|
// Check if the device has free input buffers available. If not we'll switch to the
|
|
// WAITING_FOR_INPUT_BUFFERS state, and resume encoding once we're notified buffers are
|
|
// available in the onInputBufferDone() task. Note: The input buffers are not copied into
|
|
// the device's input buffers, but rather a memory pointer is imported. We still have to
|
|
// throttle the number of enqueues queued simultaneously on the device however.
|
|
if (mInputQueue->FreeBuffersCount() == 0) {
|
|
ALOGV("Waiting for device to return input buffers");
|
|
setEncoderState(EncoderState::WAITING_FOR_INPUT_BUFFERS);
|
|
return;
|
|
}
|
|
|
|
C2ConstGraphicBlock inputBlock =
|
|
work->input.buffers.front()->data().graphicBlocks().front();
|
|
|
|
// If encoding fails, we'll wait for an event (e.g. input buffers available) to start
|
|
// encoding again.
|
|
if (!encode(inputBlock, index, timestamp)) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
// The codec 2.0 framework might queue an empty CSD request, but this is currently not
|
|
// supported. We will return the CSD with the first encoded buffer work.
|
|
// TODO(dstaessens): Avoid doing this, store CSD request work at start of output queue.
|
|
if (work->input.buffers.empty() && !endOfStream) {
|
|
ALOGV("Discarding empty CSD request");
|
|
reportWork(std::move(mInputWorkQueue.front()));
|
|
} else {
|
|
mOutputWorkQueue.push_back(std::move(mInputWorkQueue.front()));
|
|
}
|
|
mInputWorkQueue.pop();
|
|
|
|
// Drain the encoder if required.
|
|
if (endOfStream) {
|
|
drainTask(C2Component::DRAIN_COMPONENT_WITH_EOS);
|
|
}
|
|
|
|
if (mEncoderState == EncoderState::DRAINING) {
|
|
return;
|
|
} else if (mInputWorkQueue.empty()) {
|
|
setEncoderState(EncoderState::WAITING_FOR_INPUT);
|
|
return;
|
|
}
|
|
|
|
// Queue the next work item to be encoded.
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE, ::base::BindOnce(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis));
|
|
}
|
|
|
|
bool V4L2EncodeComponent::encode(C2ConstGraphicBlock block, uint64_t index, int64_t timestamp) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState == EncoderState::ENCODING);
|
|
|
|
// Update dynamic encoding parameters (bitrate, framerate, key frame) if requested.
|
|
if (!updateEncodingParameters()) return false;
|
|
|
|
mKeyFrameCounter = (mKeyFrameCounter + 1) % mKeyFramePeriod;
|
|
|
|
// If required convert the data to the V4L2 device's configured input pixel format. We
|
|
// allocate the same amount of buffers on the device input queue and the format convertor,
|
|
// so we should never run out of conversion buffers if there are free buffers in the input
|
|
// queue.
|
|
if (mInputFormatConverter) {
|
|
if (!mInputFormatConverter->isReady()) {
|
|
ALOGE("Input format convertor ran out of buffers");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
ALOGV("Converting input block (index: %" PRIu64 ")", index);
|
|
c2_status_t status = C2_CORRUPTED;
|
|
block = mInputFormatConverter->convertBlock(index, block, &status);
|
|
if (status != C2_OK) {
|
|
ALOGE("Failed to convert input block (index: %" PRIu64 ")", index);
|
|
reportError(status);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
ALOGV("Encoding input block (index: %" PRIu64 ", timestamp: %" PRId64 ", size: %dx%d)", index,
|
|
timestamp, block.width(), block.height());
|
|
|
|
// Create a video frame from the graphic block.
|
|
std::unique_ptr<InputFrame> frame = InputFrame::Create(block);
|
|
if (!frame) {
|
|
ALOGE("Failed to create video frame from input block (index: %" PRIu64
|
|
", timestamp: %" PRId64 ")",
|
|
index, timestamp);
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
// Get the video frame layout and pixel format from the graphic block.
|
|
// TODO(dstaessens) Integrate getVideoFrameLayout() into InputFrame::Create()
|
|
media::VideoPixelFormat format;
|
|
std::optional<std::vector<VideoFramePlane>> planes = getVideoFrameLayout(block, &format);
|
|
if (!planes) {
|
|
ALOGE("Failed to get input block's layout");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
if (!enqueueInputBuffer(std::move(frame), format, *planes, index, timestamp)) {
|
|
ALOGE("Failed to enqueue video frame (index: %" PRIu64 ", timestamp: %" PRId64 ")", index,
|
|
timestamp);
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
// Start streaming on the input and output queue if required.
|
|
if (!mInputQueue->IsStreaming()) {
|
|
ALOG_ASSERT(!mOutputQueue->IsStreaming());
|
|
if (!mOutputQueue->Streamon() || !mInputQueue->Streamon()) {
|
|
ALOGE("Failed to start streaming on input and output queue");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
// Start polling on the V4L2 device.
|
|
startDevicePoll();
|
|
}
|
|
|
|
// Queue all buffers on the output queue. These buffers will be used to store the encoded
|
|
// bitstreams.
|
|
while (mOutputQueue->FreeBuffersCount() > 0) {
|
|
if (!enqueueOutputBuffer()) return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void V4L2EncodeComponent::drain() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
if (mEncoderState == EncoderState::DRAINING || mEncoderState == EncoderState::ERROR) {
|
|
return;
|
|
}
|
|
|
|
ALOG_ASSERT(mInputQueue->IsStreaming() && mOutputQueue->IsStreaming());
|
|
ALOG_ASSERT(!mOutputWorkQueue.empty());
|
|
|
|
// TODO(dstaessens): Move IOCTL to device class.
|
|
struct v4l2_encoder_cmd cmd;
|
|
memset(&cmd, 0, sizeof(v4l2_encoder_cmd));
|
|
cmd.cmd = V4L2_ENC_CMD_STOP;
|
|
if (mDevice->Ioctl(VIDIOC_ENCODER_CMD, &cmd) != 0) {
|
|
ALOGE("Failed to stop encoder");
|
|
onDrainDone(false);
|
|
return;
|
|
}
|
|
ALOGV("%s(): Sent STOP command to encoder", __func__);
|
|
|
|
setEncoderState(EncoderState::DRAINING);
|
|
}
|
|
|
|
void V4L2EncodeComponent::flush() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
// Stop the device poll thread.
|
|
stopDevicePoll();
|
|
|
|
// Stop streaming on the V4L2 device, which stops all currently queued work and releases all
|
|
// buffers currently in use by the device.
|
|
// TODO(b/160540027): Calling streamoff currently results in a bug.
|
|
for (auto& queue : {mInputQueue, mOutputQueue}) {
|
|
if (queue && queue->IsStreaming() && !queue->Streamoff()) {
|
|
ALOGE("Failed to stop streaming on the device queue");
|
|
reportError(C2_CORRUPTED);
|
|
}
|
|
}
|
|
|
|
// Return all buffers to the input format convertor and clear all references to graphic blocks
|
|
// in the input queue. We don't need to clear the output map as those buffers will still be
|
|
// used.
|
|
for (auto& it : mInputBuffersMap) {
|
|
if (mInputFormatConverter && it.second) {
|
|
mInputFormatConverter->returnBlock(it.first);
|
|
}
|
|
it.second = nullptr;
|
|
}
|
|
|
|
// Report all queued work items as aborted.
|
|
std::list<std::unique_ptr<C2Work>> abortedWorkItems;
|
|
while (!mInputWorkQueue.empty()) {
|
|
std::unique_ptr<C2Work> work = std::move(mInputWorkQueue.front());
|
|
work->result = C2_NOT_FOUND;
|
|
work->input.buffers.clear();
|
|
abortedWorkItems.push_back(std::move(work));
|
|
mInputWorkQueue.pop();
|
|
}
|
|
while (!mOutputWorkQueue.empty()) {
|
|
std::unique_ptr<C2Work> work = std::move(mOutputWorkQueue.front());
|
|
work->result = C2_NOT_FOUND;
|
|
work->input.buffers.clear();
|
|
abortedWorkItems.push_back(std::move(work));
|
|
mOutputWorkQueue.pop_front();
|
|
}
|
|
if (!abortedWorkItems.empty())
|
|
mListener->onWorkDone_nb(shared_from_this(), std::move(abortedWorkItems));
|
|
|
|
// Streaming and polling on the V4L2 device input and output queues will be resumed once new
|
|
// encode work is queued.
|
|
}
|
|
|
|
std::shared_ptr<C2LinearBlock> V4L2EncodeComponent::fetchOutputBlock() {
|
|
// TODO(dstaessens): fetchLinearBlock() might be blocking.
|
|
ALOGV("Fetching linear block (size: %u)", mOutputBufferSize);
|
|
std::shared_ptr<C2LinearBlock> outputBlock;
|
|
c2_status_t status = mOutputBlockPool->fetchLinearBlock(
|
|
mOutputBufferSize,
|
|
C2MemoryUsage(C2MemoryUsage::CPU_READ |
|
|
static_cast<uint64_t>(BufferUsage::VIDEO_ENCODER)),
|
|
&outputBlock);
|
|
if (status != C2_OK) {
|
|
ALOGE("Failed to fetch linear block (error: %d)", status);
|
|
reportError(status);
|
|
return nullptr;
|
|
}
|
|
|
|
return outputBlock;
|
|
}
|
|
|
|
void V4L2EncodeComponent::onInputBufferDone(uint64_t index) {
|
|
ALOGV("%s(): Input buffer done (index: %" PRIu64 ")", __func__, index);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED);
|
|
|
|
// There are no guarantees the input buffers are returned in order, so we need to find the work
|
|
// item which this buffer belongs to.
|
|
C2Work* work = getWorkByIndex(index);
|
|
if (!work) {
|
|
ALOGE("Failed to find work associated with input buffer %" PRIu64, index);
|
|
reportError(C2_CORRUPTED);
|
|
return;
|
|
}
|
|
|
|
// We're done using the input block, release reference to return the block to the client. If
|
|
// using an input format convertor, we also need to return the block to the convertor.
|
|
LOG_ASSERT(!work->input.buffers.empty());
|
|
work->input.buffers.front().reset();
|
|
if (mInputFormatConverter) {
|
|
c2_status_t status = mInputFormatConverter->returnBlock(index);
|
|
if (status != C2_OK) {
|
|
reportError(status);
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Return all completed work items. The work item might have been waiting for it's input buffer
|
|
// to be returned, in which case we can report it as completed now. As input buffers are not
|
|
// necessarily returned in order we might be able to return multiple ready work items now.
|
|
while (!mOutputWorkQueue.empty() && isWorkDone(*mOutputWorkQueue.front())) {
|
|
reportWork(std::move(mOutputWorkQueue.front()));
|
|
mOutputWorkQueue.pop_front();
|
|
}
|
|
|
|
// We might have been waiting for input buffers to be returned after draining finished.
|
|
if (mEncoderState == EncoderState::DRAINING && mOutputWorkQueue.empty()) {
|
|
ALOGV("Draining done");
|
|
mEncoderState = EncoderState::WAITING_FOR_INPUT_BUFFERS;
|
|
}
|
|
|
|
// If we previously used up all input queue buffers we can start encoding again now.
|
|
if ((mEncoderState == EncoderState::WAITING_FOR_INPUT_BUFFERS) && !mInputWorkQueue.empty()) {
|
|
setEncoderState(EncoderState::ENCODING);
|
|
mEncoderTaskRunner->PostTask(
|
|
FROM_HERE,
|
|
::base::BindOnce(&V4L2EncodeComponent::scheduleNextEncodeTask, mWeakThis));
|
|
}
|
|
}
|
|
|
|
void V4L2EncodeComponent::onOutputBufferDone(uint32_t payloadSize, bool keyFrame, int64_t timestamp,
|
|
std::shared_ptr<C2LinearBlock> outputBlock) {
|
|
ALOGV("%s(): output buffer done (timestamp: %" PRId64 ", size: %u, key frame: %d)", __func__,
|
|
timestamp, payloadSize, keyFrame);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
if (mEncoderState == EncoderState::ERROR) {
|
|
return;
|
|
}
|
|
|
|
C2ConstLinearBlock constBlock =
|
|
outputBlock->share(outputBlock->offset(), payloadSize, C2Fence());
|
|
|
|
// If no CSD (content-specific-data, e.g. SPS for H.264) has been submitted yet, we expect this
|
|
// output block to contain CSD. We only submit the CSD once, even if it's attached to each key
|
|
// frame.
|
|
if (!mCSDSubmitted) {
|
|
ALOGV("No CSD submitted yet, extracting CSD");
|
|
std::unique_ptr<C2StreamInitDataInfo::output> csd;
|
|
C2ReadView view = constBlock.map().get();
|
|
extractCSDInfo(&csd, view.data(), view.capacity());
|
|
if (!csd) {
|
|
ALOGE("Failed to extract CSD");
|
|
reportError(C2_CORRUPTED);
|
|
return;
|
|
}
|
|
|
|
// Attach the CSD to the first item in our output work queue.
|
|
LOG_ASSERT(!mOutputWorkQueue.empty());
|
|
C2Work* work = mOutputWorkQueue.front().get();
|
|
work->worklets.front()->output.configUpdate.push_back(std::move(csd));
|
|
mCSDSubmitted = true;
|
|
}
|
|
|
|
// Get the work item associated with the timestamp.
|
|
C2Work* work = getWorkByTimestamp(timestamp);
|
|
if (!work) {
|
|
// It's possible we got an empty CSD request with timestamp 0, which we currently just
|
|
// discard.
|
|
// TODO(dstaessens): Investigate handling empty CSD requests.
|
|
if (timestamp != 0) {
|
|
reportError(C2_CORRUPTED);
|
|
}
|
|
return;
|
|
}
|
|
|
|
std::shared_ptr<C2Buffer> buffer = C2Buffer::CreateLinearBuffer(std::move(constBlock));
|
|
if (keyFrame) {
|
|
buffer->setInfo(
|
|
std::make_shared<C2StreamPictureTypeMaskInfo::output>(0u, C2Config::SYNC_FRAME));
|
|
}
|
|
work->worklets.front()->output.buffers.emplace_back(buffer);
|
|
|
|
// We can report the work item as completed if its associated input buffer has also been
|
|
// released. As output buffers are not necessarily returned in order we might be able to return
|
|
// multiple ready work items now.
|
|
while (!mOutputWorkQueue.empty() && isWorkDone(*mOutputWorkQueue.front())) {
|
|
reportWork(std::move(mOutputWorkQueue.front()));
|
|
mOutputWorkQueue.pop_front();
|
|
}
|
|
}
|
|
|
|
C2Work* V4L2EncodeComponent::getWorkByIndex(uint64_t index) {
|
|
ALOGV("%s(): getting work item (index: %" PRIu64 ")", __func__, index);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
auto it = std::find_if(mOutputWorkQueue.begin(), mOutputWorkQueue.end(),
|
|
[index](const std::unique_ptr<C2Work>& w) {
|
|
return w->input.ordinal.frameIndex.peeku() == index;
|
|
});
|
|
if (it == mOutputWorkQueue.end()) {
|
|
ALOGE("Failed to find work (index: %" PRIu64 ")", index);
|
|
return nullptr;
|
|
}
|
|
return it->get();
|
|
}
|
|
|
|
C2Work* V4L2EncodeComponent::getWorkByTimestamp(int64_t timestamp) {
|
|
ALOGV("%s(): getting work item (timestamp: %" PRId64 ")", __func__, timestamp);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(timestamp >= 0);
|
|
|
|
// Find the work with specified timestamp by looping over the output work queue. This should be
|
|
// very fast as the output work queue will never be longer then a few items. Ignore empty work
|
|
// items that are marked as EOS, as their timestamp might clash with other work items.
|
|
auto it = std::find_if(mOutputWorkQueue.begin(), mOutputWorkQueue.end(),
|
|
[timestamp](const std::unique_ptr<C2Work>& w) {
|
|
return !(w->input.flags & C2FrameData::FLAG_END_OF_STREAM) &&
|
|
w->input.ordinal.timestamp.peeku() ==
|
|
static_cast<uint64_t>(timestamp);
|
|
});
|
|
if (it == mOutputWorkQueue.end()) {
|
|
ALOGE("Failed to find work (timestamp: %" PRIu64 ")", timestamp);
|
|
return nullptr;
|
|
}
|
|
return it->get();
|
|
}
|
|
|
|
bool V4L2EncodeComponent::isWorkDone(const C2Work& work) const {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
if ((work.input.flags & C2FrameData::FLAG_END_OF_STREAM) &&
|
|
!(work.worklets.front()->output.flags & C2FrameData::FLAG_END_OF_STREAM)) {
|
|
ALOGV("Work item %" PRIu64 " is marked as EOS but draining has not finished yet",
|
|
work.input.ordinal.frameIndex.peeku());
|
|
return false;
|
|
}
|
|
|
|
if (!work.input.buffers.empty() && work.input.buffers.front()) {
|
|
ALOGV("Input buffer associated with work item %" PRIu64 " not returned yet",
|
|
work.input.ordinal.frameIndex.peeku());
|
|
return false;
|
|
}
|
|
|
|
// If the work item had an input buffer to be encoded, it should have an output buffer set.
|
|
if (!work.input.buffers.empty() && work.worklets.front()->output.buffers.empty()) {
|
|
ALOGV("Output buffer associated with work item %" PRIu64 " not returned yet",
|
|
work.input.ordinal.frameIndex.peeku());
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void V4L2EncodeComponent::reportWork(std::unique_ptr<C2Work> work) {
|
|
ALOG_ASSERT(work);
|
|
ALOGV("%s(): Reporting work item as finished (index: %llu, timestamp: %llu)", __func__,
|
|
work->input.ordinal.frameIndex.peekull(), work->input.ordinal.timestamp.peekull());
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
work->result = C2_OK;
|
|
work->workletsProcessed = static_cast<uint32_t>(work->worklets.size());
|
|
|
|
std::list<std::unique_ptr<C2Work>> finishedWorkList;
|
|
finishedWorkList.emplace_back(std::move(work));
|
|
mListener->onWorkDone_nb(shared_from_this(), std::move(finishedWorkList));
|
|
}
|
|
|
|
bool V4L2EncodeComponent::startDevicePoll() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
if (!mDevice->StartPolling(
|
|
::base::BindRepeating(&V4L2EncodeComponent::serviceDeviceTask, mWeakThis),
|
|
::base::BindRepeating(&V4L2EncodeComponent::onPollError, mWeakThis))) {
|
|
ALOGE("Device poll thread failed to start");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
ALOGV("Device poll started");
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::stopDevicePoll() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
if (!mDevice->StopPolling()) {
|
|
ALOGE("Failed to stop polling on the device");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
ALOGV("Device poll stopped");
|
|
return true;
|
|
}
|
|
|
|
void V4L2EncodeComponent::onPollError() {
|
|
ALOGV("%s()", __func__);
|
|
reportError(C2_CORRUPTED);
|
|
}
|
|
|
|
void V4L2EncodeComponent::serviceDeviceTask(bool /*event*/) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED);
|
|
|
|
if (mEncoderState == EncoderState::ERROR) {
|
|
return;
|
|
}
|
|
|
|
// Dequeue completed input (VIDEO_OUTPUT) buffers, and recycle to the free list.
|
|
while (mInputQueue->QueuedBuffersCount() > 0) {
|
|
if (!dequeueInputBuffer()) break;
|
|
}
|
|
|
|
// Dequeue completed output (VIDEO_CAPTURE) buffers, and recycle to the free list.
|
|
while (mOutputQueue->QueuedBuffersCount() > 0) {
|
|
if (!dequeueOutputBuffer()) break;
|
|
}
|
|
|
|
ALOGV("%s() - done", __func__);
|
|
}
|
|
|
|
bool V4L2EncodeComponent::enqueueInputBuffer(std::unique_ptr<InputFrame> frame,
|
|
media::VideoPixelFormat format,
|
|
const std::vector<VideoFramePlane>& planes,
|
|
int64_t index, int64_t timestamp) {
|
|
ALOGV("%s(): queuing input buffer (index: %" PRId64 ")", __func__, index);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mInputQueue->FreeBuffersCount() > 0);
|
|
ALOG_ASSERT(mEncoderState == EncoderState::ENCODING);
|
|
ALOG_ASSERT(mInputLayout->format() == format);
|
|
ALOG_ASSERT(mInputLayout->planes().size() == planes.size());
|
|
|
|
auto buffer = mInputQueue->GetFreeBuffer();
|
|
if (!buffer) {
|
|
ALOGE("Failed to get free buffer from device input queue");
|
|
return false;
|
|
}
|
|
|
|
// Mark the buffer with the frame's timestamp so we can identify the associated output buffers.
|
|
buffer->SetTimeStamp(
|
|
{.tv_sec = static_cast<time_t>(timestamp / ::base::Time::kMicrosecondsPerSecond),
|
|
.tv_usec = static_cast<time_t>(timestamp % ::base::Time::kMicrosecondsPerSecond)});
|
|
size_t bufferId = buffer->BufferId();
|
|
|
|
for (size_t i = 0; i < planes.size(); ++i) {
|
|
// Single-buffer input format may have multiple color planes, so bytesUsed of the single
|
|
// buffer should be sum of each color planes' size.
|
|
size_t bytesUsed = 0;
|
|
if (planes.size() == 1) {
|
|
bytesUsed = media::VideoFrame::AllocationSize(format, mInputLayout->coded_size());
|
|
} else {
|
|
bytesUsed = ::base::checked_cast<size_t>(
|
|
media::VideoFrame::PlaneSize(format, i, mInputLayout->coded_size()).GetArea());
|
|
}
|
|
|
|
// TODO(crbug.com/901264): The way to pass an offset within a DMA-buf is not defined
|
|
// in V4L2 specification, so we abuse data_offset for now. Fix it when we have the
|
|
// right interface, including any necessary validation and potential alignment.
|
|
buffer->SetPlaneDataOffset(i, planes[i].mOffset);
|
|
bytesUsed += planes[i].mOffset;
|
|
// Workaround: filling length should not be needed. This is a bug of videobuf2 library.
|
|
buffer->SetPlaneSize(i, mInputLayout->planes()[i].size + planes[i].mOffset);
|
|
buffer->SetPlaneBytesUsed(i, bytesUsed);
|
|
}
|
|
|
|
std::move(*buffer).QueueDMABuf(frame->getFDs());
|
|
|
|
ALOGV("Queued buffer in input queue (index: %" PRId64 ", timestamp: %" PRId64
|
|
", bufferId: %zu)",
|
|
index, timestamp, bufferId);
|
|
|
|
mInputBuffersMap[bufferId] = {index, std::move(frame)};
|
|
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::enqueueOutputBuffer() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mOutputQueue->FreeBuffersCount() > 0);
|
|
|
|
auto buffer = mOutputQueue->GetFreeBuffer();
|
|
if (!buffer) {
|
|
ALOGE("Failed to get free buffer from device output queue");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
std::shared_ptr<C2LinearBlock> outputBlock = fetchOutputBlock();
|
|
if (!outputBlock) {
|
|
ALOGE("Failed to fetch output block");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
size_t bufferId = buffer->BufferId();
|
|
|
|
std::vector<int> fds;
|
|
fds.push_back(outputBlock->handle()->data[0]);
|
|
if (!std::move(*buffer).QueueDMABuf(fds)) {
|
|
ALOGE("Failed to queue output buffer using QueueDMABuf");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
ALOG_ASSERT(!mOutputBuffersMap[bufferId]);
|
|
mOutputBuffersMap[bufferId] = std::move(outputBlock);
|
|
ALOGV("%s(): Queued buffer in output queue (bufferId: %zu)", __func__, bufferId);
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::dequeueInputBuffer() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED);
|
|
ALOG_ASSERT(mInputQueue->QueuedBuffersCount() > 0);
|
|
|
|
std::pair<bool, media::V4L2ReadableBufferRef> result = mInputQueue->DequeueBuffer();
|
|
if (!result.first) {
|
|
ALOGE("Failed to dequeue buffer from input queue");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
if (!result.second) {
|
|
// No more buffers ready to be dequeued in input queue.
|
|
return false;
|
|
}
|
|
|
|
const media::V4L2ReadableBufferRef buffer = std::move(result.second);
|
|
uint64_t index = mInputBuffersMap[buffer->BufferId()].first;
|
|
int64_t timestamp = buffer->GetTimeStamp().tv_usec +
|
|
buffer->GetTimeStamp().tv_sec * ::base::Time::kMicrosecondsPerSecond;
|
|
ALOGV("Dequeued buffer from input queue (index: %" PRId64 ", timestamp: %" PRId64
|
|
", bufferId: %zu)",
|
|
index, timestamp, buffer->BufferId());
|
|
|
|
mInputBuffersMap[buffer->BufferId()].second = nullptr;
|
|
onInputBufferDone(index);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::dequeueOutputBuffer() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(mEncoderState != EncoderState::UNINITIALIZED);
|
|
ALOG_ASSERT(mOutputQueue->QueuedBuffersCount() > 0);
|
|
|
|
std::pair<bool, media::V4L2ReadableBufferRef> result = mOutputQueue->DequeueBuffer();
|
|
if (!result.first) {
|
|
ALOGE("Failed to dequeue buffer from output queue");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
if (!result.second) {
|
|
// No more buffers ready to be dequeued in output queue.
|
|
return false;
|
|
}
|
|
|
|
media::V4L2ReadableBufferRef buffer = std::move(result.second);
|
|
size_t encodedDataSize = buffer->GetPlaneBytesUsed(0) - buffer->GetPlaneDataOffset(0);
|
|
::base::TimeDelta timestamp = ::base::TimeDelta::FromMicroseconds(
|
|
buffer->GetTimeStamp().tv_usec +
|
|
buffer->GetTimeStamp().tv_sec * ::base::Time::kMicrosecondsPerSecond);
|
|
|
|
ALOGV("Dequeued buffer from output queue (timestamp: %" PRId64
|
|
", bufferId: %zu, data size: %zu, EOS: %d)",
|
|
timestamp.InMicroseconds(), buffer->BufferId(), encodedDataSize, buffer->IsLast());
|
|
|
|
if (!mOutputBuffersMap[buffer->BufferId()]) {
|
|
ALOGE("Failed to find output block associated with output buffer");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
|
|
std::shared_ptr<C2LinearBlock> block = std::move(mOutputBuffersMap[buffer->BufferId()]);
|
|
if (encodedDataSize > 0) {
|
|
onOutputBufferDone(encodedDataSize, buffer->IsKeyframe(), timestamp.InMicroseconds(),
|
|
std::move(block));
|
|
}
|
|
|
|
// If the buffer is marked as last and we were flushing the encoder, flushing is now done.
|
|
if ((mEncoderState == EncoderState::DRAINING) && buffer->IsLast()) {
|
|
onDrainDone(true);
|
|
|
|
// Start the encoder again.
|
|
struct v4l2_encoder_cmd cmd;
|
|
memset(&cmd, 0, sizeof(v4l2_encoder_cmd));
|
|
cmd.cmd = V4L2_ENC_CMD_START;
|
|
if (mDevice->Ioctl(VIDIOC_ENCODER_CMD, &cmd) != 0) {
|
|
ALOGE("Failed to restart encoder after flushing (V4L2_ENC_CMD_START)");
|
|
reportError(C2_CORRUPTED);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Queue a new output buffer to replace the one we dequeued.
|
|
buffer = nullptr;
|
|
enqueueOutputBuffer();
|
|
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::createInputBuffers() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(!mInputQueue->IsStreaming());
|
|
ALOG_ASSERT(mInputBuffersMap.empty());
|
|
|
|
// No memory is allocated here, we just generate a list of buffers on the input queue, which
|
|
// will hold memory handles to the real buffers.
|
|
if (mInputQueue->AllocateBuffers(kInputBufferCount, V4L2_MEMORY_DMABUF) < kInputBufferCount) {
|
|
ALOGE("Failed to create V4L2 input buffers.");
|
|
return false;
|
|
}
|
|
|
|
mInputBuffersMap.resize(mInputQueue->AllocatedBuffersCount());
|
|
return true;
|
|
}
|
|
|
|
bool V4L2EncodeComponent::createOutputBuffers() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(!mOutputQueue->IsStreaming());
|
|
ALOG_ASSERT(mOutputBuffersMap.empty());
|
|
|
|
// Fetch the output block pool.
|
|
C2BlockPool::local_id_t poolId = mInterface->getBlockPoolId();
|
|
c2_status_t status = GetCodec2BlockPool(poolId, shared_from_this(), &mOutputBlockPool);
|
|
if (status != C2_OK || !mOutputBlockPool) {
|
|
ALOGE("Failed to get output block pool, error: %d", status);
|
|
return false;
|
|
}
|
|
|
|
// No memory is allocated here, we just generate a list of buffers on the output queue, which
|
|
// will hold memory handles to the real buffers.
|
|
if (mOutputQueue->AllocateBuffers(kOutputBufferCount, V4L2_MEMORY_DMABUF) <
|
|
kOutputBufferCount) {
|
|
ALOGE("Failed to create V4L2 output buffers.");
|
|
return false;
|
|
}
|
|
|
|
mOutputBuffersMap.resize(mOutputQueue->AllocatedBuffersCount());
|
|
return true;
|
|
}
|
|
|
|
void V4L2EncodeComponent::destroyInputBuffers() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(!mInputQueue->IsStreaming());
|
|
|
|
if (!mInputQueue || mInputQueue->AllocatedBuffersCount() == 0) return;
|
|
mInputQueue->DeallocateBuffers();
|
|
mInputBuffersMap.clear();
|
|
}
|
|
|
|
void V4L2EncodeComponent::destroyOutputBuffers() {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
ALOG_ASSERT(!mOutputQueue->IsStreaming());
|
|
|
|
if (!mOutputQueue || mOutputQueue->AllocatedBuffersCount() == 0) return;
|
|
mOutputQueue->DeallocateBuffers();
|
|
mOutputBuffersMap.clear();
|
|
mOutputBlockPool.reset();
|
|
}
|
|
|
|
void V4L2EncodeComponent::reportError(c2_status_t error) {
|
|
ALOGV("%s()", __func__);
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
{
|
|
std::lock_guard<std::mutex> lock(mComponentLock);
|
|
setComponentState(ComponentState::ERROR);
|
|
}
|
|
|
|
// TODO(dstaessens): Report all pending work items as finished upon failure.
|
|
if (mEncoderState != EncoderState::ERROR) {
|
|
setEncoderState(EncoderState::ERROR);
|
|
mListener->onError_nb(shared_from_this(), static_cast<uint32_t>(error));
|
|
}
|
|
}
|
|
|
|
void V4L2EncodeComponent::setComponentState(ComponentState state) {
|
|
// Check whether the state change is valid.
|
|
switch (state) {
|
|
case ComponentState::UNLOADED:
|
|
ALOG_ASSERT(mComponentState == ComponentState::LOADED);
|
|
break;
|
|
case ComponentState::LOADED:
|
|
ALOG_ASSERT(mComponentState == ComponentState::UNLOADED ||
|
|
mComponentState == ComponentState::RUNNING ||
|
|
mComponentState == ComponentState::ERROR);
|
|
break;
|
|
case ComponentState::RUNNING:
|
|
ALOG_ASSERT(mComponentState == ComponentState::LOADED);
|
|
break;
|
|
case ComponentState::ERROR:
|
|
break;
|
|
}
|
|
|
|
ALOGV("Changed component state from %s to %s", componentStateToString(mComponentState),
|
|
componentStateToString(state));
|
|
mComponentState = state;
|
|
}
|
|
|
|
void V4L2EncodeComponent::setEncoderState(EncoderState state) {
|
|
ALOG_ASSERT(mEncoderTaskRunner->RunsTasksInCurrentSequence());
|
|
|
|
// Check whether the state change is valid.
|
|
switch (state) {
|
|
case EncoderState::UNINITIALIZED:
|
|
// TODO(dstaessens): Check all valid state changes.
|
|
break;
|
|
case EncoderState::WAITING_FOR_INPUT:
|
|
ALOG_ASSERT(mEncoderState == EncoderState::UNINITIALIZED ||
|
|
mEncoderState == EncoderState::ENCODING ||
|
|
mEncoderState == EncoderState::DRAINING);
|
|
break;
|
|
case EncoderState::WAITING_FOR_INPUT_BUFFERS:
|
|
ALOG_ASSERT(mEncoderState == EncoderState::ENCODING);
|
|
break;
|
|
case EncoderState::ENCODING:
|
|
ALOG_ASSERT(mEncoderState == EncoderState::WAITING_FOR_INPUT ||
|
|
mEncoderState == EncoderState::WAITING_FOR_INPUT_BUFFERS ||
|
|
mEncoderState == EncoderState::DRAINING);
|
|
break;
|
|
case EncoderState::DRAINING:
|
|
ALOG_ASSERT(mEncoderState == EncoderState::ENCODING);
|
|
break;
|
|
case EncoderState::ERROR:
|
|
break;
|
|
}
|
|
|
|
ALOGV("Changed encoder state from %s to %s", encoderStateToString(mEncoderState),
|
|
encoderStateToString(state));
|
|
mEncoderState = state;
|
|
}
|
|
|
|
const char* V4L2EncodeComponent::componentStateToString(V4L2EncodeComponent::ComponentState state) {
|
|
switch (state) {
|
|
case ComponentState::UNLOADED:
|
|
return "UNLOADED";
|
|
case ComponentState::LOADED:
|
|
return "LOADED";
|
|
case ComponentState::RUNNING:
|
|
return "RUNNING";
|
|
case ComponentState::ERROR:
|
|
return "ERROR";
|
|
}
|
|
}
|
|
|
|
const char* V4L2EncodeComponent::encoderStateToString(V4L2EncodeComponent::EncoderState state) {
|
|
switch (state) {
|
|
case EncoderState::UNINITIALIZED:
|
|
return "UNINITIALIZED";
|
|
case EncoderState::WAITING_FOR_INPUT:
|
|
return "WAITING_FOR_INPUT";
|
|
case EncoderState::WAITING_FOR_INPUT_BUFFERS:
|
|
return "WAITING_FOR_INPUT_BUFFERS";
|
|
case EncoderState::ENCODING:
|
|
return "ENCODING";
|
|
case EncoderState::DRAINING:
|
|
return "Draining";
|
|
case EncoderState::ERROR:
|
|
return "ERROR";
|
|
}
|
|
}
|
|
|
|
} // namespace android
|