Skip to content

Commit

Permalink
Merge pull request #295 from Eyevinn/handle-avc-pictime
Browse files Browse the repository at this point in the history 
Handle avc pictiming with cbpDbpDelay present
  • Loading branch information
@tobbee
tobbee authored Nov 17, 2023
2 parents 86b6ec4 + f8c606d commit 547a03b
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 22 deletions.
6 changes: 5 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

- Nothing yet
### Fixed

- Test of AVC PicTiming SEI with cbpDbpDelay set
- mp4ff-nallister has nicer output for annexb streams
- mp4ff-nallister handles AVC PicTiming SEI with cbpDbpDelay set

## [0.40.1] - 2023-11-01

Expand Down
105 changes: 91 additions & 14 deletions cmd/mp4ff-nallister/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,31 @@ func main() {
if err != nil {
log.Fatal(err)
}
if *codec == "avc" {
err = printAVCNalus(nalus, 0, 0, *seiLevel, *parameterSets, *printRaw)
} else {
err = printHEVCNalus(nalus, 0, 0, *seiLevel, *parameterSets, *printRaw)
}
frames, err := findAnnexBFrames(nalus, *codec)
if err != nil {
log.Fatal(err)
}
var avcSPS *avc.SPS
for i, frame := range frames {
if *codec == "avc" {
if avcSPS == nil {
for _, nalu := range frame {
if avc.GetNaluType(nalu[0]) == avc.NALU_SPS {
avcSPS, err = avc.ParseSPSNALUnit(nalu, true)
if err != nil {
log.Fatal(err)
}
}
}
}
err = printAVCNalus(avcSPS, frame, i+1, 0, *seiLevel, *parameterSets, *printRaw)
} else {
err = printHEVCNalus(frame, i+1, 0, *seiLevel, *parameterSets, *printRaw)
}
if err != nil {
log.Fatal(err)
}
}
os.Exit(0)
}

Expand Down Expand Up @@ -109,18 +126,27 @@ func main() {
func parseProgressiveMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel int, parameterSets bool, nrRaw int) error {
videoTrak, ok := findFirstVideoTrak(f.Moov)
if !ok {
return fmt.Errorf("No video track found")
return fmt.Errorf("no video track found")
}

var avcSPS *avc.SPS
var err error
stbl := videoTrak.Mdia.Minf.Stbl
if stbl.Stsd.AvcX != nil {
codec = "avc"
if stbl.Stsd.AvcX.AvcC != nil {
avcSPS, err = avc.ParseSPSNALUnit(stbl.Stsd.AvcX.AvcC.SPSnalus[0], true)
if err != nil {
return fmt.Errorf("error parsing SPS: %s", err)
}
}
} else if stbl.Stsd.HvcX != nil {
codec = "hevc"
}
nrSamples := stbl.Stsz.SampleNumber
mdat := f.Mdat
mdatPayloadStart := mdat.PayloadAbsoluteOffset()

for sampleNr := 1; sampleNr <= int(nrSamples); sampleNr++ {
chunkNr, sampleNrAtChunkStart, err := stbl.Stsc.ChunkNrFromSampleNr(sampleNr)
if err != nil {
Expand All @@ -145,11 +171,21 @@ func parseProgressiveMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel i
}
switch codec {
case "avc", "h.264", "h264":
err = printAVCNalus(nalus, sampleNr, decTime+uint64(cto), seiLevel, parameterSets, nrRaw)
if avcSPS == nil {
for _, nalu := range nalus {
if avc.GetNaluType(nalu[0]) == avc.NALU_SPS {
avcSPS, err = avc.ParseSPSNALUnit(nalu, true)
if err != nil {
return fmt.Errorf("error parsing SPS: %s", err)
}
}
}
}
err = printAVCNalus(avcSPS, nalus, sampleNr, decTime+uint64(cto), seiLevel, parameterSets, nrRaw)
case "hevc", "h.265", "h265":
err = printHEVCNalus(nalus, sampleNr, decTime+uint64(cto), seiLevel, parameterSets, nrRaw)
default:
return fmt.Errorf("Unknown codec: %s", codec)
return fmt.Errorf("unknown codec: %s", codec)
}
if err != nil {
return err
Expand Down Expand Up @@ -184,15 +220,23 @@ func getChunkOffset(stbl *mp4.StblBox, chunkNr int) int64 {

func parseFragmentedMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel int, parameterSets bool, nrRaw int) error {
var trex *mp4.TrexBox
var avcSPS *avc.SPS
var err error
if f.Init != nil { // Auto-detect codec if moov box is there
moov := f.Init.Moov
videoTrak, ok := findFirstVideoTrak(moov)
if !ok {
return fmt.Errorf("No video track found")
return fmt.Errorf("no video track found")
}
stbl := videoTrak.Mdia.Minf.Stbl
if stbl.Stsd.AvcX != nil {
codec = "avc"
if stbl.Stsd.AvcX.AvcC != nil {
avcSPS, err = avc.ParseSPSNALUnit(stbl.Stsd.AvcX.AvcC.SPSnalus[0], true)
if err != nil {
return fmt.Errorf("error parsing SPS: %s", err)
}
}
} else if stbl.Stsd.HvcX != nil {
codec = "hevc"
}
Expand All @@ -215,11 +259,11 @@ func parseFragmentedMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel in
}
switch codec {
case "avc", "h.264", "h264":
err = printAVCNalus(nalus, i+1, s.PresentationTime(), seiLevel, parameterSets, nrRaw)
err = printAVCNalus(avcSPS, nalus, i+1, s.PresentationTime(), seiLevel, parameterSets, nrRaw)
case "hevc", "h.265", "h265":
err = printHEVCNalus(nalus, i+1, s.PresentationTime(), seiLevel, parameterSets, nrRaw)
default:
return fmt.Errorf("Unknown codec: %s", codec)
return fmt.Errorf("unknown codec: %s", codec)
}

if err != nil {
Expand All @@ -232,11 +276,10 @@ func parseFragmentedMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel in
return nil
}

func printAVCNalus(nalus [][]byte, nr int, pts uint64, seiLevel int, parameterSets bool, nrRaw int) error {
func printAVCNalus(avcSPS *avc.SPS, nalus [][]byte, nr int, pts uint64, seiLevel int, parameterSets bool, nrRaw int) error {
msg := ""
var seiNALUs [][]byte
totLen := 0
var avcSPS *avc.SPS
for i, nalu := range nalus {
totLen += 4 + len(nalu)
if i > 0 {
Expand All @@ -249,7 +292,7 @@ func printAVCNalus(nalus [][]byte, nr int, pts uint64, seiLevel int, parameterSe
case avc.NALU_SPS:
avcSPS, err = avc.ParseSPSNALUnit(nalu, true)
if err != nil {
return fmt.Errorf("Error parsing SPS: %s", err)
return fmt.Errorf("error parsing SPS: %s", err)
}
case avc.NALU_NON_IDR, avc.NALU_IDR:
sliceType, err := avc.GetSliceTypeFromNALU(nalu)
Expand Down Expand Up @@ -389,3 +432,37 @@ func bytesToStringN(data []byte, maxNrBytes int) string {
}
return hex.EncodeToString(data)
}

func findAnnexBFrames(nalus [][]byte, codec string) ([][][]byte, error) {
var isAUD func([]byte) bool
switch codec {
case "avc":
isAUD = isAvcAudNalu
case "hevc":
isAUD = isHEVCAudNalu
default:
return nil, fmt.Errorf("unknown codec: %s", codec)
}
var frames [][][]byte
frameStart := 0
for i, nalu := range nalus {
if isAUD(nalu) {
if i > frameStart {
frames = append(frames, nalus[frameStart:i])
frameStart = i
}
}
}
if frameStart < len(nalus) {
frames = append(frames, nalus[frameStart:])
}
return frames, nil
}

func isAvcAudNalu(nalu []byte) bool {
return avc.GetNaluType(nalu[0]) == avc.NALU_AUD
}

func isHEVCAudNalu(nalu []byte) bool {
return hevc.GetNaluType(nalu[0]) == hevc.NALU_AUD
}
1 change: 1 addition & 0 deletions sei/sei1.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ func DecodePicTimingAvcSEI(sd *SEIData) (SEIMessage, error) {

// DecodePicTimingAvcSEIHRD decodes AVC SEI message 1 PicTiming with HRD parameters.
// cbpDbpDelay length fields must be properly set if cbpDbpDelay is not nil.
// The delay values in cbpDbpDelay will then be set by the decoder by reading the bits.
// It is assumed that pict_struct_present_flag is true, so that a 4-bit pict_struct value is present.
func DecodePicTimingAvcSEIHRD(sd *SEIData, cbpDbpDelay *CbpDbpDelay, timeOffsetLen byte) (SEIMessage, error) {
buf := bytes.NewBuffer(sd.Payload())
Expand Down
42 changes: 35 additions & 7 deletions sei/sei_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,46 +282,68 @@ const (
seiCEA608Hex = "0434b500314741393403cefffc9420fc94aefc9162fce56efc67bafc91b9" +
"fcb0b0fcbab0fcb0bafcb031fcbab0fcb080fc942cfc942f80"
seiAVCMulti = "0001c001061b0509b8000080"
seiAVCPicTiming = "010f00011a00000300090c2e268a000003004080"
missingRbspTrailingBits = "01061b0509b80000"
seiHEVCMulti = "000a8000000300403dc017a6900105040000be05880660404198b41080"
seiHEVCHDR = "891800000300000300000300000300000300000300000300000300000300000300000300009004000003000080"
)

type avcHRD struct {
cbpDelay *sei.CbpDbpDelay
timeOffsetLen byte
}

func TestParseSEI(t *testing.T) {

testCases := []struct {
name string
codec sei.Codec
naluHex string
avcHRD *avcHRD
wantedTypes []uint
wantedStrings []string
expNonFatalErr error
}{
{"AVC multi", sei.AVC, seiAVCMulti, []uint{0, 1},
{"AVC PicTiming", sei.AVC, seiAVCPicTiming,
&avcHRD{
cbpDelay: &sei.CbpDbpDelay{
InitialCpbRemovalDelayLengthMinus1: 23,
CpbRemovalDelayLengthMinus1: 23,
DpbOutputDelayLengthMinus1: 23,
},
timeOffsetLen: 24,
},
[]uint{1},
[]string{
`SEIPicTimingType (1), size=15, time=20:40:09:46 offset=24`,
},
nil,
},
{"AVC multi", sei.AVC, seiAVCMulti, nil, []uint{0, 1},
[]string{
`SEIBufferingPeriodType (0), size=1, "c0"`,
`SEIPicTimingType (1), size=6, time=00:00:46:09 offset=0`,
},
nil,
},
{"Missing RBSP Trailing Bits", sei.AVC, missingRbspTrailingBits, []uint{1},
{"Missing RBSP Trailing Bits", sei.AVC, missingRbspTrailingBits, nil, []uint{1},
[]string{
`SEIPicTimingType (1), size=6, time=00:00:46:09 offset=0`,
},
sei.ErrRbspTrailingBitsMissing,
},
{"Type 0", sei.AVC, sei0Hex, []uint{0}, []string{`SEIBufferingPeriodType (0), size=7, "810f1c00507440"`}, nil},
{"CEA-608", sei.AVC, seiCEA608Hex, []uint{4},
{"Type 0", sei.AVC, sei0Hex, nil, []uint{0}, []string{`SEIBufferingPeriodType (0), size=7, "810f1c00507440"`}, nil},
{"CEA-608", sei.AVC, seiCEA608Hex, nil, []uint{4},
[]string{`SEI type 4 CEA-608, size=52, field1: "942094ae9162e56e67ba91b9b0b0bab0b0bab031bab0b080942c942f", field2: ""`}, nil},
{"HEVC multi", sei.HEVC, seiHEVCMulti, []uint{0, 1, 136},
{"HEVC multi", sei.HEVC, seiHEVCMulti, nil, []uint{0, 1, 136},
[]string{
`SEIBufferingPeriodType (0), size=10, "80000000403dc017a690"`,
`SEIPicTimingType (1), size=5, "040000be05"`,
`SEITimeCodeType (136), size=6, time=13:49:12:08 offset=0`,
},
nil,
},
{"Type HDR HEVC", sei.HEVC, seiHEVCHDR, []uint{137, 144},
{"Type HDR HEVC", sei.HEVC, seiHEVCHDR, nil, []uint{137, 144},
[]string{
"SEIMasteringDisplayColourVolumeType (137) 24B: primaries=(0, 0) (0, 0) (0, 0)," +
" whitePoint=(0, 0), maxLum=0, minLum=0",
Expand All @@ -344,7 +366,13 @@ func TestParseSEI(t *testing.T) {
t.Errorf("%s: Not %d but %d sei messages found", tc.name, len(tc.wantedStrings), len(seis))
}
for i := range seis {
seiMessage, err := sei.DecodeSEIMessage(&seis[i], tc.codec)
var seiMessage sei.SEIMessage
if tc.avcHRD == nil {
seiMessage, err = sei.DecodeSEIMessage(&seis[i], tc.codec)
} else {
seiMessage, err = sei.DecodePicTimingAvcSEIHRD(&seis[i], tc.avcHRD.cbpDelay, tc.avcHRD.timeOffsetLen)

}
if err != nil {
t.Error(err)
}
Expand Down

0 comments on commit 547a03b

Please sign in to comment.