/* * Copyright (C) 2014-2026 Amlogic, Inc. All rights reserved. * * All information contained herein is Amlogic confidential. * * This software is provided to you pursuant to Software License Agreement * (SLA) with Amlogic Inc ("Amlogic"). This software may be used * only in accordance with the terms of this agreement. * * Redistribution and use in source and binary forms, with or without * modification is strictly prohibited without prior written permission from * Amlogic. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define LOG_TAG "Presentation" #include "Presentation.h" #include #include #include #include #include #include #include #include #include #include #include #include "AndroidHidlRemoteRender.h" #include "ParserFactory.h" #include "RenderFactory.h" #include "StreamUtils.h" #define MAX_ALLOWED_EXT_SPU_NUM 1000 #define MAX_ALLOWED_STREAM_SPU_NUM 10 #define MAX_ALLOWED_QUEUED_MEM (5 * 1920 * 1080 * 4) #define DEFAULT_DELAY_TIME 2 // second #define MAX_SHOW_DIFF_PTS_TIME_MS 2000 // These devices are used to get the frc_delay time, this delay time is used to // adjust the subtitle presentation time. #define DEV_FRC "/dev/frc" #define FRC_IOC_GET_VIDEO_LATENCY _IOR('F', 0x03, unsigned int) #define DEV_DPSS "/dev/dpss0" #define DPSS_IOC_GET_VIDEO_LATENCY _IOR('M', 0x03, unsigned int) static const int64_t ADDJUST_SPU_END_PTS_MS = 4 * 1000LL; // The subtitle is quite short for displaying if the display time less than 30ms, // user can adjust this value for user experience. static const int MINIMUM_REMAIN_DISPLAY_TIME_MS = 30; #define INVALID_PTS 0x1ffffffff namespace { static size_t totalQueuedMemSize(std::list> &list) { size_t size = 0; // Calculate the size of all subtitles in the cache if (list.size() > 0) { std::for_each(list.begin(), list.end(), [&](std::shared_ptr &s) { size += s->buffer_size; }); } return size; } static bool cmpSpu(std::shared_ptr a, std::shared_ptr b) { return a->pts < b->pts; } // Typically, only one spu, but some case maybe 2 or 3 static std::shared_ptr getShowingSpuFromList( std::list> &list) { if (list.size() == 0) { return nullptr; } std::shared_ptr spu = list.front(); if (!spu->isSimpleText) { return list.back(); } // For multiline, combine all the subtitles. if (list.size() > 1) { int totalSize = 0; std::for_each(list.begin(), list.end(), [&](std::shared_ptr &s) { totalSize += s->buffer_size + 1; // Add "\n" as below strcat. }); auto newSpu = std::shared_ptr(new AML_SPUVAR()); memcpy(newSpu.get(), list.front().get(), sizeof(AML_SPUVAR)); newSpu->useMalloc = false; newSpu->buffer_size = totalSize + 1; // additional '\0' newSpu->spu_data = new uint8_t[newSpu->buffer_size](); std::for_each(list.begin(), list.end(), [&](std::shared_ptr &s) { strcat((char *)newSpu->spu_data, (const char *)s->spu_data); strcat((char *)newSpu->spu_data, "\n"); }); newSpu->spu_data[totalSize] = 0; spu = newSpu; } return spu; } } // namespace Presentation::Presentation(bool registeredOnDataCB) : mRegisteredOnDataCB(registeredOnDataCB) { SUBTITLE_LOGI_FUNC_IN(); std::unique_lock autolock(mMutex); mMsgProcess = nullptr; // only access in threadloop. } Presentation::~Presentation() { SUBTITLE_LOGI_FUNC_IN(); // TODO: do we need poke thread exit immediately? by post a message? // then need add a lock, for protect access mLooper in multi-thread std::unique_lock autolock(mMutex); if (mMsgProcess != nullptr) { // delete mMsgProcess; // We do not delete here, let sp pointer do this! mMsgProcess->join(); mMsgProcess = nullptr; // late sp delete it. } // Housekeeping, especially for direct render in Linux if (mRender) { resetSubtitleItem(); mRender.reset(); } } bool Presentation::notifyStartTimeStamp(int64_t startpts) { mOrigStartTimeModifier = ms2ns(pts2Ms(startpts)); mStartTimeModifier = mOrigStartTimeModifier; SUBTITLE_LOGI("startpts(%" PRId64 ") --> mStartTimeModifier(%" PRId64 " ms)", startpts, ns2ms(mStartTimeModifier)); if (mParser) { mParser->notifyRenderStartTimestamp(startpts); } return true; } bool Presentation::syncCurrentPresentTime(int64_t pts) { // amnumediaplayer may report 33bit invalid pts, we must filter INVALID_PTS. if (INVALID_PTS == pts) { SUBTITLE_LOGE("got invalid pts"); mCurrentPresentRelativeTime = mStartPresentMonoTimeNs = -1; return false; } auto diffPresentTimeMs = ns2ms(mCurrentPresentRelativeTime); mCurrentPresentRelativeTime = ms2ns(pts2Ms(pts)); diffPresentTimeMs = ns2ms(mCurrentPresentRelativeTime) - diffPresentTimeMs; if (mParser) { mParser->notifyRenderTimeChanged(pts); } mStartPresentMonoTimeNs = systemTime(SYSTEM_TIME_MONOTONIC) - mCurrentPresentRelativeTime; // Log information, which is reported in each frame, so don't log it so frequently. static int i = 0; if (i++ % 100 == 0) { SUBTITLE_LOGI("pts= %" PRId64 " (%" PRId64 " ms, diffPts= %" PRId64 " ms), mStartPresentMonoTime= %" PRId64 " ms", pts, ns2ms(mCurrentPresentRelativeTime), diffPresentTimeMs, ns2ms(mStartPresentMonoTimeNs)); } // External subtitle, just polling and showing the subtitle. if (mParser != nullptr && mParser->isExternalSub()) { std::unique_lock autolock(mMutex); if (mMsgProcess != nullptr) { mMsgProcess->notifyMessage(MessageProcess::MSG_PTS_TIME_CHECK_SPU); } } return true; } bool Presentation::startPresent(std::shared_ptr parser) { SUBTITLE_LOGI_FUNC_IN(); if (parser == nullptr) { SUBTITLE_LOGE("Error! parser is nullptr"); return false; } mParser = parser; mParser->notifyRenderStartTimestamp(ms2Pts(ns2ms(mStartTimeModifier))); mRender = std::move(RenderFactory::create(mParser->getParseType(), mRegisteredOnDataCB)); // Initialize the show state if (mRender) { show(); } { // mMutex is for mMsgProcess. // Don't lock mParser as which has its own lock // std::unique_lock autolock(mMutex); std::unique_lock autolock(mMutex); mMsgProcess = new MessageProcess(this, parser->isExternalSub()); } mLatestFrcTimeMs = 0; mIsFrcDelayTimeDoubleConfirmed = false; mFrcUpdateClock = 0; return true; } bool Presentation::stopPresent() { SUBTITLE_LOGI_FUNC_IN(); if (mAiTranslation) { mAiTranslation.reset(); } { std::unique_lock autolock(mMutex); if (mMsgProcess != nullptr) { mMsgProcess->join(); mMsgProcess->decStrong(nullptr); mMsgProcess = nullptr; } } mParser.reset(); SUBTITLE_LOGI_FUNC_OUT(); return true; } // ASS/SSA subtitle may have two continuous packets which have the same pts. bool Presentation::combineSamePtsSubtitle(std::shared_ptr spu1, std::shared_ptr spu2) { if (!spu1 || !spu2) { SUBTITLE_LOGE("spu1 or spu2 is invalid"); return false; } if (spu1->isExtSub || spu1->isImmediatePresent || !spu1->isSimpleText) { SUBTITLE_LOGE("spu1 validation failed. isExtSub=%d, isImmediatePresent=%d, isSimpleText=%d", spu1->isExtSub, spu1->isImmediatePresent, spu1->isSimpleText); return false; } if (spu2->isExtSub || spu2->isImmediatePresent || !spu2->isSimpleText) { SUBTITLE_LOGE("spu2 validation failed. isExtSub=%d, isImmediatePresent=%d, isSimpleText=%d", spu2->isExtSub, spu2->isImmediatePresent, spu2->isSimpleText); return false; } if (spu1->pts != spu2->pts) { SUBTITLE_LOGE("the pts of spu1 and spu2 are different"); return false; } if (!spu1->spu_data || !spu2->spu_data) { SUBTITLE_LOGE("spu1->spu_data or spu2->spu_data is NULL"); return false; } int dataLen1 = spu1->buffer_size; int dataLen2 = spu2->buffer_size; if (dataLen1 <= 0 || dataLen2 <= 0) { SUBTITLE_LOGE("invalid buffer size dataLen1=%d, dataLen2=%d", dataLen1, dataLen2); return false; } int newSize = dataLen1 + 1 + dataLen2 + 1; SUBTITLE_LOGI("combine pts:%" PRId64 ", spu1 size:%d, spu2 size:%d, new buffer size:%d", spu1->pts, dataLen1, dataLen2, newSize); uint8_t *newBuffer = new uint8_t[newSize](); strncpy((char *)newBuffer, (char *)spu1->spu_data, dataLen1); strncat((char *)newBuffer, "\n", 1); strncat((char *)newBuffer, (char *)spu2->spu_data, dataLen2); if (spu1->useMalloc) { free(spu1->spu_data); } else { delete[] spu1->spu_data; } spu1->spu_data = newBuffer; spu1->buffer_size = newSize; spu1->useMalloc = false; return true; } bool Presentation::resetForSeek() { SUBTITLE_LOGI_FUNC_IN(); mLatestFrcTimeMs = 0; mIsFrcDelayTimeDoubleConfirmed = false; mFrcUpdateClock = 0; std::unique_lock autolock(mMutex); if (mMsgProcess != nullptr) { mMsgProcess->notifyMessage(MessageProcess::MSG_RESET_MESSAGE_QUEUE); } if (mAiTranslation) { mAiTranslation->resetForSeek(); } return true; } void Presentation::notifySubdataAdded() { // External subtitle file is loaded at the playback begin, and the presentation // is triggered by vPts, so stop trigger the presentation when new SPU was added, // it makes useless loop. if (mParser && mParser->isExternalSub()) { return; } std::unique_lock autolock(mMutex); if (mMsgProcess != nullptr) { mMsgProcess->notifyMessage(MessageProcess::MSG_PTS_TIME_CHECK_SPU); } } bool Presentation::show() { if (mRender != nullptr) { return mRender->show(); } return false; } bool Presentation::hide() { if (mRender != nullptr) { return mRender->hide(); } return false; } bool Presentation::setSubTranslationLanguage(const std::string &srcLang, const std::string &dstLang) { SUBTITLE_LOGI("srcLang=%s, srcLang=%s", srcLang.empty() ? " " : srcLang.c_str(), dstLang.empty() ? " " : dstLang.c_str()); if (mAiTranslation) { resetForSeek(); } else { mAiTranslation = std::make_unique(*this); if (!mAiTranslation) { SUBTITLE_LOGE("fail to create AI translation"); return false; } } if (!srcLang.empty() && !mAiTranslation->isLanguageAvailable(srcLang)) { SUBTITLE_LOGE("language %s isn't supported", srcLang.c_str()); return false; } if (!dstLang.empty() && !mAiTranslation->isLanguageAvailable(dstLang)) { SUBTITLE_LOGE("language %s isn't supported", dstLang.c_str()); return false; } return mAiTranslation->loadAaiLanguage(srcLang, dstLang); } // ISubtitleAiTranslationObserver void Presentation::onReceiveTranslatedText(const std::shared_ptr &item) { if (!item) { SUBTITLE_LOGE("unexpected null spu"); return; } if (!mRender) { SUBTITLE_LOGE("render is stopped"); return; } if (!mParser) { SUBTITLE_LOGE("parser is stopped"); return; } showSubtitleItem(item, mParser->getParseType()); } void Presentation::onError(const std::string &reason) { SUBTITLE_LOGE("%s", reason.c_str()); } void Presentation::send2RenderDisplay(std::shared_ptr spu) { if (!spu) { SUBTITLE_LOGE("unexpected null spu"); return; } if (!mRender) { SUBTITLE_LOGE("render is null"); return; } if (mAiTranslation) { // AI translation is only for SUBTITLE_TEXT_DISPLAY if (spu->spu_data && spu->buffer_size > 0 && spu->spu_width == 0 && spu->spu_height == 0) { mAiTranslation->push2Translate(spu); return; } } if (!mParser) { SUBTITLE_LOGE("parser is null"); return; } showSubtitleItem(spu, mParser->getParseType()); } void Presentation::showSubtitleItem(const std::shared_ptr &spu, int type) { std::unique_lock autolock(mRenderMutex); if (!mRender) { SUBTITLE_LOGE("render is null"); return; } mRender->showSubtitleItem(spu, type); } void Presentation::hideSubtitleItem(const std::shared_ptr &spu) { std::unique_lock autolock(mRenderMutex); if (!mRender) { SUBTITLE_LOGE("render is null"); return; } mRender->hideSubtitleItem(spu); } void Presentation::removeSubtitleItem(const std::shared_ptr &spu) { std::unique_lock autolock(mRenderMutex); if (!mRender) { SUBTITLE_LOGE("render is null"); return; } mRender->removeSubtitleItem(spu); } void Presentation::resetSubtitleItem() { std::unique_lock autolock(mRenderMutex); if (!mRender) { SUBTITLE_LOGE("render is null"); return; } mRender->resetSubtitleItem(); } void Presentation::adjustStartTimeByFrcIfNeed() { // Force update FRC latency time one more time as which is stable in 1 second after playback. if (!mIsFrcDelayTimeDoubleConfirmed && mFrcUpdateClock > 0 && getCurrentTimeMs() - mFrcUpdateClock >= 1000) { mLatestFrcTimeMs = 0; } if (mLatestFrcTimeMs > 0 || mIsFrcDelayTimeDoubleConfirmed) { return; } // Step 1: Get the FRC delay time. mLatestFrcTimeMs = getFrcDelayTimeMs(); if (mLatestFrcTimeMs == -2) { // Check if the target doesn't support MEMC function. if (mFrcUpdateClock > 0) { mIsFrcDelayTimeDoubleConfirmed = true; } else { mFrcUpdateClock = getCurrentTimeMs(); } return; } // MEMC is not ready or something is wrong. if (mLatestFrcTimeMs <= 0) { if (mLatestFrcTimeMs < 0) { SUBTITLE_LOGI("warning: invalid FRC time %d", mLatestFrcTimeMs); } mFrcUpdateClock = getCurrentTimeMs(); return; } // Step 2: update the presentation start time. auto oldStartTimeMs = ns2ms(mStartTimeModifier); mStartTimeModifier = mOrigStartTimeModifier - ms2ns(mLatestFrcTimeMs); SUBTITLE_LOGI("start time was adjusted from %" PRId64 " ms to %" PRId64 " ms", oldStartTimeMs, ns2ms(mStartTimeModifier)); // Need update the time one more time after 1 second later since the driver // need a little time to make the time stable. if (mFrcUpdateClock > 0) { mIsFrcDelayTimeDoubleConfirmed = true; } mFrcUpdateClock = getCurrentTimeMs(); if (mParser && mLatestFrcTimeMs > 0) { mParser->notifyRenderStartTimestamp(ms2Pts(ns2ms(mStartTimeModifier))); } } int Presentation::getFrcDelayTimeMs() { unsigned int videoLatency = 0; int fd = -1; int ret = -1; // Step 1: try getting videoLatency from device DEV_FRC fd = open(DEV_FRC, O_RDWR); if (fd >= 0) { ret = ioctl(fd, FRC_IOC_GET_VIDEO_LATENCY, &videoLatency); if (ret >= 0) { if (videoLatency > 0) { SUBTITLE_LOGI("got MEMC delay time %d ms from %s", videoLatency, DEV_FRC); } close(fd); return videoLatency; } SUBTITLE_LOGE("fail to get frc_delay from %s, error=%m", DEV_FRC); close(fd); } // Step 2: if step 1 is failed, try getting videoLatency from device DEV_DPSS fd = open(DEV_DPSS, O_RDWR); if (fd >= 0) { ret = ioctl(fd, DPSS_IOC_GET_VIDEO_LATENCY, &videoLatency); if (ret >= 0) { if (videoLatency > 0) { SUBTITLE_LOGI("got MEMC delay time %d ms from %s", videoLatency, DEV_DPSS); } close(fd); return videoLatency; } SUBTITLE_LOGE("fail to get frc_delay from %s, error=%m", DEV_DPSS); close(fd); } // Step 3: if both DEV_FRC and DEV_DPSS are not available, // it means MEMC is not available in the target. SUBTITLE_LOGI("warning: MEMC function is not available on this target %m"); // Use -2 here as the driver may return -1 if something is wrong there. return -2; } void Presentation::dump(int fd, const char *prefix) { dprintf(fd, "%s Presentation:\n", prefix); dprintf(fd, "%s CurrentPresentRelativeTime[dvb time]: %" PRId64 "\n", prefix, ms2Pts(ns2ms(mCurrentPresentRelativeTime))); dprintf(fd, "%s StartPresentMonoTime[dvb time]: %" PRId64 "\n", prefix, ms2Pts(ns2ms(mStartPresentMonoTimeNs))); dprintf(fd, "%s StartTimeModifier[dvb time]: %" PRId64 "\n", prefix, ms2Pts(ns2ms(mStartTimeModifier))); dprintf(fd, "\n"); if (mParser != nullptr) { dprintf(fd, "%s Parser : %p\n", prefix, mParser.get()); } if (mRender != nullptr) { dprintf(fd, "%s Render : %p\n", prefix, mRender.get()); } dprintf(fd, "\n"); { dprintf(fd, "%s Ready for showing SPUs:\n", prefix); auto it = mEmittedShowingSpu.begin(); for (; it != mEmittedShowingSpu.end(); ++it) { if ((*it) != nullptr) { (*it)->dump(fd, " "); } } } dprintf(fd, "\n"); { dprintf(fd, "%s Showed but not deleted SPUs:\n", prefix); auto it = mEmittedFaddingSpu.begin(); for (; it != mEmittedFaddingSpu.end(); ++it) { if ((*it) != nullptr) { (*it)->dump(fd, " "); } } } } Presentation::MessageProcess::MessageProcess(Presentation *present, bool isExtSub) { SUBTITLE_LOGI_FUNC_IN(); mRequestThreadExit = false; mPresent = present; mIsExtSub = isExtSub; // hold a reference for RefBase object // we move the inc here, before the thread started, to avoid multi-thread problem incStrong(nullptr); mLooperThread = std::shared_ptr(new std::thread(&MessageProcess::looperLoop, this)); } Presentation::MessageProcess::~MessageProcess() { mLastShowingSpu = nullptr; mPresent = nullptr; SUBTITLE_LOGI_FUNC_OUT(); } void Presentation::MessageProcess::join() { mRequestThreadExit = true; if (mLooper != nullptr) { mLooper->removeMessages(this, MSG_PTS_TIME_CHECK_SPU); mLooper->wake(); } if (mLooperThread->joinable()) { mLooperThread->join(); } else { SUBTITLE_LOGE("_exception"); } mLooper = nullptr; } bool Presentation::MessageProcess::notifyMessage(int what) { if (mLooper != nullptr) { mLooper->sendMessage(this, Message(what)); } return true; } void Presentation::MessageProcess::handleMessage(const Message &message) { // We sync from video pts. but some player not start video // when decoded and present subtitle. so we need wait video pts. // Don't need wait for video pts if the SPU need to be presented immediately. if (mPresent->mCurrentPresentRelativeTime <= 0 && !mPresent->mParser->isImmediatePresent()) { SUBTITLE_LOGI("media sync is not started, wait 200ms ..."); mLooper->sendMessageDelayed(ms2ns(200), this, Message(MSG_PTS_TIME_CHECK_SPU)); return; } mPresent->adjustStartTimeByFrcIfNeed(); return mIsExtSub ? handleExtSub(message) : handleStreamSub(message); } static std::list> computeShowingSpuList( std::list> &list, int64_t timestamp) { std::list> showingList; std::shared_ptr spuLess; std::shared_ptr spuBig; // spu list was already sorted by pts for (auto it : list) { auto pts = ms2ns(pts2Ms(it->pts)); auto ptsEnd = ms2ns(pts2Ms(it->m_delay)); if (timestamp >= pts && timestamp < ptsEnd) { showingList.push_back(it); if (it->dynGen == true) { return showingList; } // To check if the next spu which is a showing spu continue; } if (showingList.size() > 0 && timestamp > ptsEnd) { break; } } // nullptr means no subtitle is displayed on the screen. return showingList; } void Presentation::MessageProcess::handleExtSub(const Message &message) { switch (message.what) { case MSG_PTS_TIME_CHECK_SPU: { // External subtitle parser always decode all the subtitle items, // syncCurrentPresentTime triggers the display. mLooper->removeMessages(this, MSG_PTS_TIME_CHECK_SPU); if (mRequestThreadExit) { SUBTITLE_LOGE("message process is exiting"); return; } if (mPresent->mParser == nullptr) { SUBTITLE_LOGE("parser is nullptr"); return; } // 1. Collect all decoded items, save them into mEmittedShowingSpu. std::shared_ptr spu; auto &spuList = mPresent->mEmittedShowingSpu; auto starTimeMs = getCurrentTimeMs(); while ((spu = mPresent->mParser->tryConsumeDecodedItem()) != nullptr) { if (spu->m_delay <= 0 || spu->m_delay <= spu->pts + ms2Pts(500)) { if (spu->buffer_size > 0) { spu->m_delay = spu->pts + ms2Pts(ADDJUST_SPU_END_PTS_MS); } else { spu->m_delay = spu->pts + ms2Pts(100); } } spuList.push_back(spu); spuList.sort(cmpSpu); // It takes time to load all external subtitles, to be sure not block process, // break current loading and let the next round to continue. auto spendTimeMs = getCurrentTimeMs() - starTimeMs; if (spendTimeMs >= 1000) { SUBTITLE_LOGI("loaded %zu SPUs took %" PRId64 " ms", spuList.size(), spendTimeMs); break; } } // 2. Find showing SPUs by pts, none showing spu means no subtitles // are displayed on the screen // Current video playback time unit is ns. const auto timestampNs = mPresent->mStartTimeModifier + mPresent->mCurrentPresentRelativeTime; auto showingSpuList = computeShowingSpuList(spuList, timestampNs); if (showingSpuList.size() > 0) { spu = getShowingSpuFromList(showingSpuList); if (showingSpuList.size() > 1) { // Log here to trace multiple lines. int i = 0; for (const auto &it : showingSpuList) { if (it->spu_data) { SUBTITLE_LOGI("[%d]: %s", ++i, it->spu_data); } } if (spu->spu_data) { SUBTITLE_LOGI("final: %s", spu->spu_data); } } // Be sure the remain display time is more than 33ms(one frame time). // Don't do it in computeShowingSpuList as this is a special case checking. // // NOTE: m_delay was already adjusted to be sure which is not 0. auto remainMs = pts2Ms(spu->m_delay) - ns2ms(timestampNs); if (mLastShowingSpu != spu && remainMs >= MINIMUM_REMAIN_DISPLAY_TIME_MS) { SUBTITLE_LOGI("Show SPU: TimeStamp:%" PRId64 " ms(%s), SPU[pts:%" PRId64 " ms + %" PRId64 " ms, ptsDiff=%" PRId64 " ms, remainMs=%" PRId64 " ms) data: %s (size=%d)]", ns2ms(timestampNs), ms2ClockString(ns2ms(timestampNs)).c_str(), pts2Ms(spu->pts), pts2Ms(spu->m_delay - spu->pts), pts2Ms(spu->pts) - ns2ms(timestampNs), remainMs, spu->spu_data, spu->buffer_size); mPresent->mEmittedFaddingSpu.clear(); mPresent->send2RenderDisplay(spu); } mLastShowingSpu = spu; } else if (mLastShowingSpu != nullptr) { mPresent->resetSubtitleItem(); mLastShowingSpu = nullptr; } } break; case MSG_RESET_MESSAGE_QUEUE: { SUBTITLE_LOGI("MSG_RESET_MESSAGE_QUEUE"); mPresent->mEmittedFaddingSpu.clear(); mPresent->resetSubtitleItem(); } break; default: break; } } // Stream sub decode and show the subtitle when received data . void Presentation::MessageProcess::handleStreamSub(const Message &message) { switch (message.what) { case MSG_PTS_TIME_CHECK_SPU: { mLooper->removeMessages(this, MSG_PTS_TIME_CHECK_SPU); if (mRequestThreadExit) { SUBTITLE_LOGE("message process is exiting"); return; } if (mPresent == nullptr || mPresent->mParser == nullptr) { SUBTITLE_LOGE("mPresent or parser is nullptr"); return; } std::shared_ptr spu = mPresent->mParser->tryConsumeDecodedItem(); const auto timestampNs = mPresent->mStartTimeModifier + mPresent->mCurrentPresentRelativeTime; if (timestampNs < 0) { SUBTITLE_LOGE("Unexpected VPts (TimeStamp=%" PRId64 " ms, StartTime=%" PRId64 " ms)", ns2ms(mPresent->mCurrentPresentRelativeTime), ns2ms(mPresent->mStartTimeModifier)); } int64_t ptsNs = 0; if (spu != nullptr) { ptsNs = ms2ns(pts2Ms(spu->pts)); // Workaround to present the exception subtitle PTS. // Subtitle parser should assure subtitle's PTS is parsed correctly, but subtitle // PES's PTS may be zero in some private TS or wrong TS streams, this is a // workaround to present such "wrong" SPU 1 second later. auto ptsDiffNs = ptsNs > timestampNs ? (ptsNs - timestampNs) : (timestampNs - ptsNs); if (ptsDiffNs >= ms2ns(200 * 1000)) { SUBTITLE_LOGE("Got SPU: unexpected SPU ptsDiff(%" PRId64 ") >= 200s, adjust the pts from %" PRId64 " to timeStamp+1second %" PRId64 " ms", ns2ms(ptsDiffNs), ns2ms(ptsNs), ns2ms(timestampNs) + 1000); auto ptsDiffOfDelay = spu->m_delay - spu->pts; ptsNs = timestampNs + ms2ns(1000); spu->pts = ms2Pts(ns2ms(ptsNs)); spu->m_delay = spu->pts + ptsDiffOfDelay; } if (TYPE_SUBTITLE_CLOSED_CAPTION == spu->subtitle_type || (timestampNs != 0 || spu->pts == 0)) { mPresent->mEmittedShowingSpu.push_back(spu); mPresent->mEmittedShowingSpu.sort(cmpSpu); } SUBTITLE_LOGI("Got SPU: TimeStamp=%" PRId64 " ms, StartTime=%" PRId64 " ms, SPU[objectId=%d pts:%" PRId64 "(%" PRId64 " d%" PRId64 " ms) m_delay:%" PRId64 "(%" PRId64 " ms) data:%p(%d) type:%d]," " isImmediate=%d, ShowingSpu=%zu", ns2ms(mPresent->mCurrentPresentRelativeTime), ns2ms(mPresent->mStartTimeModifier), spu->objectSegmentId, spu->pts, pts2Ms(spu->pts), pts2Ms(spu->pts) - ns2ms(timestampNs), spu->m_delay, pts2Ms(spu->m_delay), spu->spu_data, spu->buffer_size, spu->subtitle_type, spu->isImmediatePresent, mPresent->mEmittedShowingSpu.size()); // Keep a maximum showing spu limit here to same memory. /* NOTE: leave these statements here for different strategy int maxAllowedItem = spu->isBitmapSpu() ? MAX_ALLOWED_STREAM_SPU_NUM : MAX_ALLOWED_STREAM_SPU_NUM*50; while (mPresent->mEmittedShowingSpu.size() > maxAllowedItem) { mPresent->mEmittedShowingSpu.pop_front(); }*/ if (spu->buffer_size + totalQueuedMemSize(mPresent->mEmittedShowingSpu) >= MAX_ALLOWED_QUEUED_MEM) { SUBTITLE_LOGI( "Warning! The memory size occupied by the total queue has exceeded the " "maximum value."); size_t halfQueueSize = mPresent->mEmittedShowingSpu.size() / 2; for (size_t i = 0; i < halfQueueSize; ++i) { mPresent->mEmittedShowingSpu.pop_front(); } } } // Handle showing // ================= if (mPresent->mEmittedShowingSpu.size() > 0) { spu = mPresent->mEmittedShowingSpu.front(); if (spu != nullptr && TYPE_SUBTITLE_PGS != spu->subtitle_type) { SUBTITLE_LOGI( "isExtSub=%d, TimeStamp=%" PRId64 " ms, StartTime=%" PRId64 " ms, SPU[pts=%" PRId64 "(%" PRId64 " d%" PRId64 " ms) m_delay:(%" PRId64 " ms d%" PRId64 " ms)], ShowingSpu=%zu, FaddingSpu=%zu", spu->isExtSub, ns2ms(mPresent->mCurrentPresentRelativeTime), ns2ms(mPresent->mStartTimeModifier), spu->pts, pts2Ms(spu->pts), pts2Ms(spu->pts) - ns2ms(timestampNs), pts2Ms(spu->m_delay), pts2Ms(spu->m_delay) - ns2ms(timestampNs), mPresent->mEmittedShowingSpu.size(), mPresent->mEmittedFaddingSpu.size()); } // In case seek done, then throw out-of-date subtitle while (spu != nullptr && spu->isExtSub && spu->m_delay > 0 && ns2ms(timestampNs) >= pts2Ms(spu->m_delay)) { mPresent->mEmittedShowingSpu.pop_front(); spu = mPresent->mEmittedShowingSpu.front(); // If the spu is nullptr then sendmessage to get subtitle spu if (spu == nullptr) { mLooper->sendMessageDelayed(ms2ns(100), this, Message(MSG_PTS_TIME_CHECK_SPU)); break; } } // Throw cached 10min later spus. mPresent->mEmittedShowingSpu.remove_if( [×tampNs](std::shared_ptr spu) { if (ns2ms(timestampNs) + 10 * 60 * 1000LL <= pts2Ms(spu->pts) && !spu->isImmediatePresent) { SUBTITLE_LOGI("delete[timestamp:%" PRId64 "ms, pts=%" PRId64 " ms], data=%p", ns2ms(timestampNs), pts2Ms(spu->pts), spu->spu_data); return true; } return false; }); if (mPresent->mEmittedShowingSpu.size() == 0) { SUBTITLE_LOGE("all items in emitted spu are out of date, ignore!"); return; } if (spu != nullptr) { ptsNs = ms2ns(pts2Ms(spu->pts)); // If decoded pts is null, and is normal subtitle(not immediate display), // show it. if (spu->pts <= 0 && !spu->isImmediatePresent) { spu->pts = ms2Pts(ns2ms(timestampNs)); spu->m_delay = spu->pts + ms2Pts(DEFAULT_DELAY_TIME * 1000); ptsNs = ms2ns(pts2Ms(spu->pts)); } // Fix fadding time, if not valid. // Note: PGS has its own presentation solution, the delay time is calculated // correctly in it. if (spu->isImmediatePresent && (spu->m_delay <= 0 || spu->m_delay <= spu->pts)) { // immediatePresent has no pts, this may affect the fading calculate. // assign a pts and fading delay pts. spu->pts = ms2Pts(ns2ms(timestampNs)); spu->m_delay = spu->pts + ms2Pts(ADDJUST_SPU_END_PTS_MS); ptsNs = ms2ns(pts2Ms(spu->pts)); } else if (spu->m_delay <= 0 || spu->m_delay <= spu->pts) { spu->m_delay = spu->pts + ms2Pts(ADDJUST_SPU_END_PTS_MS); } const int64_t toleranceNs = 33 * 1000 * 1000LL; // 33 ms tolerance const int64_t ptsShowDiffNs = (timestampNs + toleranceNs) - ptsNs; // spu is presented in case of: // 1. spu is tagged as isImmediatePresent; // 2. spu is expired in MAX_SHOW_DIFF_PTS_TIME_MS(2000) ms; // 3. spu is expired more than 2000ms, but the end time is still valid. if (spu->isImmediatePresent || (ptsShowDiffNs >= 0 && ptsShowDiffNs <= ms2ns(MAX_SHOW_DIFF_PTS_TIME_MS)) || (ptsShowDiffNs >= 0 && spu->m_delay != 0 && ms2ns(pts2Ms(spu->m_delay)) >= timestampNs)) { mPresent->mEmittedShowingSpu.pop_front(); if (mPresent->mEmittedShowingSpu.size() > 0) { std::shared_ptr secondSpu = mPresent->mEmittedShowingSpu.front(); if (mPresent->combineSamePtsSubtitle(spu, secondSpu)) { mPresent->mEmittedShowingSpu.pop_front(); } } SUBTITLE_LOGI("Show SPU: TimeStamp=%" PRId64 " ms, StartTime=%" PRId64 " ms, SPU[objectId=%d pts=%" PRId64 "(%" PRId64 " d%" PRId64 " ms) m_delay=(%" PRId64 " ms d%" PRId64 " ms) data=%p(%d) isImmediate=%d]," " ptsShowDiff=%" PRId64 " ms, ShowingSpu=%zu, FaddingSpu=%zu", ns2ms(mPresent->mCurrentPresentRelativeTime), ns2ms(mPresent->mStartTimeModifier), spu->objectSegmentId, spu->pts, pts2Ms(spu->pts), pts2Ms(spu->pts) - ns2ms(timestampNs), pts2Ms(spu->m_delay), pts2Ms(spu->m_delay) - ns2ms(timestampNs), spu->spu_data, spu->buffer_size, spu->isImmediatePresent, ns2ms(ptsShowDiffNs), mPresent->mEmittedShowingSpu.size(), mPresent->mEmittedFaddingSpu.size()); if (spu->spu_data == nullptr) { // null SPU means need clear the subtitle on the screen, let the next // SPU refresh the previous if which have the same PTS, otherwise, there // is a flicker when clearing and display. if (mPresent->mEmittedShowingSpu.empty()) { mPresent->hideSubtitleItem(spu); } else { auto &it = mPresent->mEmittedShowingSpu.front(); if (it->pts == spu->pts && it->spu_data != nullptr) { SUBTITLE_LOGI( "Show SPU: found the next SPU has the same pts with current"); } else { mPresent->hideSubtitleItem(spu); } } } else { mPresent->send2RenderDisplay(spu); } // Add to fadding list, post for fadding std::shared_ptr cachedSpu = mPresent->mEmittedFaddingSpu.front(); if (cachedSpu != nullptr) { mPresent->mEmittedFaddingSpu.pop_front(); mPresent->removeSubtitleItem(cachedSpu); } // ATSC CLOSED CAPTION now don't hide according to hide spu // but according to showing spu json data.so exclude CLOSED CAPTION. if (TYPE_SUBTITLE_CLOSED_CAPTION != spu->subtitle_type) { mPresent->mEmittedFaddingSpu.push_back(spu); } else { // ATSC CC has 8 window json data at most continuously(sarnoff test). // If test video fps is 30 and through notifySubAdded operation // will show slow or lost every 33ms. // so need trigger the next round presentation quickly mLooper->sendMessageDelayed(ms2ns(15), this, Message(MSG_PTS_TIME_CHECK_SPU)); } } else if (ptsShowDiffNs > ms2ns(MAX_SHOW_DIFF_PTS_TIME_MS) && (spu->m_delay != 0 && ms2ns(pts2Ms(spu->m_delay)) <= timestampNs)) { // Both spu's pts and end pts are expired, it may happen in seek case, // remove such spu if both spu's start and end pts are expired. SUBTITLE_LOGE( "Remove SPU: SPU pts is expired, ptsShowDiff=%" PRId64 " ms TimeStamp:%" PRId64 " StartTime=%" PRId64 " ms, SPU[pts:%" PRId64 "(%" PRId64 " ms)] mEmittedFaddingSpu=%zu, mEmittedShowingSpu=%zu", ns2ms(ptsShowDiffNs), ns2ms(mPresent->mCurrentPresentRelativeTime), ns2ms(mPresent->mStartTimeModifier), spu->pts, pts2Ms(spu->pts), mPresent->mEmittedFaddingSpu.size(), mPresent->mEmittedShowingSpu.size()); mPresent->mEmittedFaddingSpu.clear(); mPresent->resetSubtitleItem(); // Removed the expired spu and fire message MSG_PTS_TIME_CHECK_SPU to // trigger the next round. mPresent->mEmittedShowingSpu.pop_front(); mLooper->sendMessageDelayed(ms2ns(5), this, Message(MSG_PTS_TIME_CHECK_SPU)); } else { int64_t messageTimerNs = spu->isExtSub ? ms2ns(100) : ptsNs - timestampNs; if (messageTimerNs <= 0) { SUBTITLE_LOGE("the expired spu should already be handled, pts:%" PRId64 "ms m_delay=%" PRId64 "ms timestampMs: %" PRId64, pts2Ms(spu->pts), pts2Ms(spu->m_delay), ns2ms(timestampNs)); } mLooper->sendMessageDelayed(messageTimerNs, this, Message(MSG_PTS_TIME_CHECK_SPU)); } } else if (spu == nullptr) { SUBTITLE_LOGE("should not be nullptr when showing"); } } // Handle fading // ================= if (mPresent->mEmittedFaddingSpu.size() > 0) { spu = mPresent->mEmittedFaddingSpu.front(); if (spu != nullptr && !spu->isKeepShowing) { auto delayedNs = ms2ns(pts2Ms(spu->m_delay)); auto aheadDelayNs = ms2ns(spu->isExtSub ? 5 * 1000 : 100 * 1000); if (timestampNs != 0 && ptsNs <= timestampNs && delayedNs <= timestampNs) { mPresent->mEmittedFaddingSpu.pop_front(); SUBTITLE_LOGI("1 fade SPU: TimeStamp=%" PRId64 " ms, StartTime=%" PRId64 " ms, SPU[objectId=%d pts=%" PRId64 "(%" PRId64 " ms) m_delay:%" PRId64 "(%" PRId64 " ms) data:%p(%d), isKeepShowing:%d, " "isImmediatePresent:%d, isTtxSubtitle:%d]", ns2ms(mPresent->mCurrentPresentRelativeTime), ns2ms(mPresent->mStartTimeModifier), spu->objectSegmentId, spu->pts, pts2Ms(spu->pts), spu->m_delay, pts2Ms(spu->m_delay), spu->spu_data, spu->buffer_size, spu->isKeepShowing, spu->isImmediatePresent, spu->isTtxSubtitle); if (spu->isKeepShowing == false) { mPresent->hideSubtitleItem(spu); } else { mPresent->removeSubtitleItem(spu); } } else if (timestampNs != 0 && delayedNs - timestampNs > aheadDelayNs) { // when the video gets to begin,to get rid of the subtitle data to avoid the // memory leak because when pull out the cable , the video pts became zero. // And the timestamp became zero. And then it would clear the subtitle data // queue which may be used by the dtvkit.It may cause crash as the "bad file // description". so add the "(timestamp != 0)" condition check. mPresent->mEmittedFaddingSpu.pop_front(); SUBTITLE_LOGI("2 fade SPU: TimeStamp=%" PRId64 "ms, StartTime=%" PRId64 " ms, ItemPts=%" PRId64 "(%" PRId64 "), m_delay=%" PRId64 "(%" PRId64 "), data:%p, delayed:%" PRId64 ", aheadDelayNs:%" PRIu64, ns2ms(mPresent->mCurrentPresentRelativeTime), ns2ms(mPresent->mStartTimeModifier), spu->pts, pts2Ms(spu->pts), spu->m_delay, pts2Ms(spu->m_delay), spu->spu_data, delayedNs, ns2ms(aheadDelayNs)); if (spu->isKeepShowing == false) { mPresent->hideSubtitleItem(spu); } else { mPresent->removeSubtitleItem(spu); } } // SPU's end pts is untrusted, we need trigger the next round presentation // quickly, otherwise, the untrusted SPU end pts may block the next // mEmittedShowingSpu. if (mPresent->mEmittedShowingSpu.size() > 0 || mPresent->mEmittedFaddingSpu.size() > 0) { mLooper->sendMessageDelayed(ms2ns(15), this, Message(MSG_PTS_TIME_CHECK_SPU)); } } else if (spu == nullptr) { SUBTITLE_LOGE("should not be nullptr when fadding"); } } } break; case MSG_RESET_MESSAGE_QUEUE: { SUBTITLE_LOGI("MSG_RESET_MESSAGE_QUEUE"); mPresent->mEmittedShowingSpu.clear(); mPresent->mEmittedFaddingSpu.clear(); mPresent->resetSubtitleItem(); } break; default: break; } } void Presentation::MessageProcess::looperLoop() { mLooper = new Looper(false); if (mPresent->mCurrentPresentRelativeTime <= 0) { mLooper->sendMessageDelayed(ms2ns(100), this, Message(MSG_PTS_TIME_CHECK_SPU)); } else { mLooper->sendMessageDelayed(100LL, this, Message(MSG_PTS_TIME_CHECK_SPU)); } mPresent->mEmittedShowingSpu.clear(); mPresent->mEmittedFaddingSpu.clear(); while (!mRequestThreadExit) { int32_t ret = mLooper->pollAll(500); switch (ret) { case android::Looper::POLL_WAKE: SUBTITLE_LOGI("A_LOOPER_POLL_WAKE"); break; case android::Looper::POLL_TIMEOUT: break; default: SUBTITLE_LOGI("A_LOOPER_POLL_ERROR ret=%d", ret); break; } } SUBTITLE_LOGI("looperLoop EXITED"); }