Good morning. Our C++ CustomH264Sink derived classes' afterGettingFrame
methods shown below are not recognizing the IFrame bytes supposed to be
contained in a H264 packet generated from a file containing 5 minutes of
Live555 H264 video data. Please tell me what we are doing incorrectly or
forgetting to do.
The "testRTSPClient" demo application receives each (video and/or audio) frame
into a memory buffer, but does not do anything with the frame data. You can,
however, use this code as a model for a 'media player' application that decodes
and renders these frames. Note, in particular, the "DummySink" class that the
"testRTSPClient" demo application uses - and the (non-static)
"DummySink::afterGettingFrame()" function. When this function is called, a
complete 'frame' (for H.264 or H.265, this will be a "NAL unit") will have
already been delivered into "fReceiveBuffer". Note that our "DummySink"
implementation doesn't actually do anything with this data; that's why it's
called a 'dummy' sink.
If you want to decode (or otherwise process) these frames, you would replace
"DummySink" with your own "MediaSink" subclass. Its "afterGettingFrame()"
function would pass the data (at "fReceiveBuffer", of length "frameSize") to a
decoder
If you are receiving H.264 video data, there is one more thing that you have
to do before you start feeding frames to your decoder. H.264 streams have
out-of-band configuration information ("SPS" and "PPS" NAL units) that you may
need to feed to the decoder to initialize it. To get this information, call
"MediaSubsession::fmtp_spropparametersets()" (on the video 'subsession'
object). This will give you a (ASCII) character string. You can then pass this
to "parseSPropParameterSets()" (defined in the file
"include/H264VideoRTPSource.hh"), to generate binary NAL units for your
decoder.
void CustomH264Sink::afterGettingFrame(void* clientData, unsigned frameSize,
unsigned numTruncatedBytes, struct timeval presentationTime, unsigned
durationInMicroseconds)
{
CustomH264Sink* sink = (CustomH264Sink*)clientData;
/* HERE I DISCOVERED THAT sink byte buffer does not contain an IFRAME */
sink->afterGettingFrame(frameSize, numTruncatedBytes, presentationTime,
durationInMicroseconds);
}
void CustomH264Sink::afterGettingFrame(unsigned frameSize, unsigned
numTruncatedBytes, struct timeval presentationTime, unsigned
/*durationInMicroseconds*/)
{
// We've just received a frame of data. (Optionally) print out
information about it:
if (fSubsession.rtpSource()->curPacketMarkerBit())
{
m_nFrameSize += frameSize;
int nNALType = fReceiveBuffer[H264_NAL_HEADER_STARTCODE_LENGTH] &
0x1F;
bool bNalHeaderPresent = false;
if ((nNALType == 0) && (m_nFrameSize >=
H264_NAL_HEADER_STARTCODE_LENGTH + H264_NAL_HEADER_STARTCODE_LENGTH))
{ // Most of the cameras sending the frames without nal
header start code, however some cameras send the frame with start code
// Since we initialized the buffer with start code, we need
to skip that.
// We are reinitilzing without nal header start code, we
will execute this code only for the first frame
nNALType = fReceiveBuffer[H264_NAL_HEADER_STARTCODE_LENGTH +
H264_NAL_HEADER_STARTCODE_LENGTH] & 0x1F;
// First make sure we have valid NAL Type
if (nNALType == NALTYPE::NALTYPE_IDRPicture || nNALType ==
NALTYPE::NALTYPE_SEI || nNALType == NALTYPE::NALTYPE_SequenceParameterSet
|| nNALType == NALTYPE::NALTYPE_PictureParameterSet ||
nNALType == NALTYPE::NALTYPE_AccessUnitDelimiter || nNALType ==
NALTYPE::NALTYPE_SliceLayerWithoutPartitioning)
{
memmove(fReceiveBuffer, fReceiveBuffer +
H264_NAL_HEADER_STARTCODE_LENGTH, m_nFrameSize -
H264_NAL_HEADER_STARTCODE_LENGTH);
bNalHeaderPresent = true;
}
}
if (nNALType == NALTYPE::NALTYPE_IDRPicture)
{
envir() << "I Frame (" << m_nFrameWidth << "x" <<
m_nFrameHeight << ") " << m_nFrameSize + frameSize << "\n";
m_nFrameCounter = 0;
if (m_nConfigLength)
m_pCamera->Add2FrameQueue(CC_SAMPLETYPE_MPEG4AVC_CONFIG, m_pConfig,
m_nConfigLength, m_nFrameWidth, m_nFrameHeight, m_nFrameCounter);
m_pCamera->Add2FrameQueue(CC_SAMPLETYPE_MPEG4AVC_IFRAME,
fReceiveBuffer, m_nFrameSize, m_nFrameWidth, m_nFrameHeight, m_nFrameCounter);
DisplayDiagnosticsInfo(m_nFrameSize);
CalculateFrameRate();
CalculateIFrameInterval(true);
}
else if (nNALType == NALTYPE::NALTYPE_SEI)
{
//envir() << "SEI " << m_nFrameSize << "\n";
}
else if (nNALType == NALTYPE::NALTYPE_SequenceParameterSet)
{
// Some Cameras send I frame with Config Data , process
those cases
unsigned long nEndOfConfigData = m_nFrameSize + 1;
unsigned long nBytesToCheck = m_nFrameSize - 5;
BYTE nNALTypeSPS;
const unsigned char* pData = (const unsigned
char*)fReceiveBuffer;
// Special Case 1
for (unsigned long n = 0; n < nBytesToCheck; ++n)
{
if (pData[0] == 0x00 && pData[1] == 0x00 && pData[2]
== 0x00 && pData[3] == 0x01)
{
nNALTypeSPS = pData[4] & 0x1F;
if ((nNALTypeSPS == NALTYPE_IDRPicture) ||
(nNALTypeSPS == NALTYPE_SliceLayerWithoutPartitioning))
{ // Frame Data is started
nEndOfConfigData = n;
break;
}
}
++pData;
}
if (nEndOfConfigData < m_nFrameSize)
{
m_nFrameCounter = 0;
if (m_nConfigLength)
m_pCamera->Add2FrameQueue(CC_SAMPLETYPE_MPEG4AVC_CONFIG, fReceiveBuffer,
nEndOfConfigData, m_nFrameWidth, m_nFrameHeight, m_nFrameCounter);
m_pCamera->Add2FrameQueue(CC_SAMPLETYPE_MPEG4AVC_IFRAME, fReceiveBuffer +
nEndOfConfigData, m_nFrameSize - nEndOfConfigData, m_nFrameWidth,
m_nFrameHeight, m_nFrameCounter);
DisplayDiagnosticsInfo(m_nFrameSize);
CalculateFrameRate();
CalculateIFrameInterval(true);
}
else
{ // This is normal case, most of the cameras
if (m_pConfig == NULL && m_nFrameSize)
{
m_nConfigLength = m_nFrameSize;
m_pConfig = new uint8_t[m_nConfigLength];
memcpy(m_pConfig, fReceiveBuffer,
m_nConfigLength);
}
}
// Diag
//envir() << "SPS " << m_nFrameSize << "\n";
}
else if (nNALType == NALTYPE::NALTYPE_PictureParameterSet)
{
}
else if (nNALType == NALTYPE::NALTYPE_AccessUnitDelimiter)
{
//envir() << "AUD " << m_nFrameSize << "\n";
}
else if (nNALType ==
NALTYPE::NALTYPE_SliceLayerWithoutPartitioning)
{
//envir() << "P Frame " << m_nFrameSize << "\n";
m_nFrameCounter++;
m_pCamera->Add2FrameQueue(CC_SAMPLETYPE_MPEG4AVC_PFRAME,
fReceiveBuffer, m_nFrameSize, m_nFrameWidth, m_nFrameHeight, m_nFrameCounter);
CalculateFrameRate();
CalculateIFrameInterval(false);
}
else
{
envir() << "Unrecognizable DATA
............................................................. \n";
envir() << nNALType;
bNalHeaderPresent = false;
}
//envir() << m_nFrameSize << "\n";
if (bNalHeaderPresent)
{ // Reinitilize the offset, since this camera is sending nal
header start code
m_nNalHeaderStartCodeOffset = 0;
}
m_nFrameSize = m_nNalHeaderStartCodeOffset;
}
else
{
m_nFrameSize += frameSize;
// 08/19/2015 ODD Case, Major IP Camera Vendor's firmware sends
SEI data without rtp market set, Needs to investigate further
int nNALType = fReceiveBuffer[H264_NAL_HEADER_STARTCODE_LENGTH] &
0x1F;
if (nNALType == NALTYPE::NALTYPE_SEI)
{
// Ignore the packet, we don't need SEI motion detection data
m_nFrameSize = H264_NAL_HEADER_STARTCODE_LENGTH;
envir() << "SEI without Marker " << m_nFrameSize << "
}
//envir() << "Incomplete Data....................... \n";
}
// Then continue, to request the next frame of data:
continuePlaying();
}
Boolean CustomH264Sink::continuePlaying()
{
if (fSource == NULL) return False; // sanity check (should not happen)
// Request the next frame of data from our input source.
"afterGettingFrame()" will get called later, when it arrives:
#if 1
fSource->getNextFrame(fReceiveBuffer + m_nFrameSize,
fReceiveBufferLength - m_nFrameSize,
afterGettingFrame, this,
onSourceClosure, this);
#else
fSource->getNextFrame(fReceiveBuffer, fReceiveBufferLength,
afterGettingFrame, this,
onSourceClosure, this);
#endif
return True;
}
Any help is greatly appreciated.
_______________________________________________
live-devel mailing list
live-devel@lists.live555.com
http://lists.live555.com/mailman/listinfo/live-devel
