diff --git a/CHANGELOG.md b/CHANGELOG.md index b339a4d7..7321d0a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- Nothing yet +### Fixed + +- Test of AVC PicTiming SEI with cbpDbpDelay set +- mp4ff-nallister has nicer output for annexb streams +- mp4ff-nallister handles AVC PicTiming SEI with cbpDbpDelay set ## [0.40.1] - 2023-11-01 diff --git a/cmd/mp4ff-nallister/main.go b/cmd/mp4ff-nallister/main.go index 9bba6493..25895073 100644 --- a/cmd/mp4ff-nallister/main.go +++ b/cmd/mp4ff-nallister/main.go @@ -68,14 +68,31 @@ func main() { if err != nil { log.Fatal(err) } - if *codec == "avc" { - err = printAVCNalus(nalus, 0, 0, *seiLevel, *parameterSets, *printRaw) - } else { - err = printHEVCNalus(nalus, 0, 0, *seiLevel, *parameterSets, *printRaw) - } + frames, err := findAnnexBFrames(nalus, *codec) if err != nil { log.Fatal(err) } + var avcSPS *avc.SPS + for i, frame := range frames { + if *codec == "avc" { + if avcSPS == nil { + for _, nalu := range frame { + if avc.GetNaluType(nalu[0]) == avc.NALU_SPS { + avcSPS, err = avc.ParseSPSNALUnit(nalu, true) + if err != nil { + log.Fatal(err) + } + } + } + } + err = printAVCNalus(avcSPS, frame, i+1, 0, *seiLevel, *parameterSets, *printRaw) + } else { + err = printHEVCNalus(frame, i+1, 0, *seiLevel, *parameterSets, *printRaw) + } + if err != nil { + log.Fatal(err) + } + } os.Exit(0) } @@ -109,18 +126,27 @@ func main() { func parseProgressiveMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel int, parameterSets bool, nrRaw int) error { videoTrak, ok := findFirstVideoTrak(f.Moov) if !ok { - return fmt.Errorf("No video track found") + return fmt.Errorf("no video track found") } + var avcSPS *avc.SPS + var err error stbl := videoTrak.Mdia.Minf.Stbl if stbl.Stsd.AvcX != nil { codec = "avc" + if stbl.Stsd.AvcX.AvcC != nil { + avcSPS, err = avc.ParseSPSNALUnit(stbl.Stsd.AvcX.AvcC.SPSnalus[0], true) + if err != nil { + return fmt.Errorf("error parsing SPS: %s", err) + } + } } else if stbl.Stsd.HvcX != nil { codec = "hevc" } nrSamples := stbl.Stsz.SampleNumber mdat := f.Mdat mdatPayloadStart := mdat.PayloadAbsoluteOffset() + for sampleNr := 1; sampleNr <= int(nrSamples); sampleNr++ { chunkNr, sampleNrAtChunkStart, err := stbl.Stsc.ChunkNrFromSampleNr(sampleNr) if err != nil { @@ -145,11 +171,21 @@ func parseProgressiveMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel i } switch codec { case "avc", "h.264", "h264": - err = printAVCNalus(nalus, sampleNr, decTime+uint64(cto), seiLevel, parameterSets, nrRaw) + if avcSPS == nil { + for _, nalu := range nalus { + if avc.GetNaluType(nalu[0]) == avc.NALU_SPS { + avcSPS, err = avc.ParseSPSNALUnit(nalu, true) + if err != nil { + return fmt.Errorf("error parsing SPS: %s", err) + } + } + } + } + err = printAVCNalus(avcSPS, nalus, sampleNr, decTime+uint64(cto), seiLevel, parameterSets, nrRaw) case "hevc", "h.265", "h265": err = printHEVCNalus(nalus, sampleNr, decTime+uint64(cto), seiLevel, parameterSets, nrRaw) default: - return fmt.Errorf("Unknown codec: %s", codec) + return fmt.Errorf("unknown codec: %s", codec) } if err != nil { return err @@ -184,15 +220,23 @@ func getChunkOffset(stbl *mp4.StblBox, chunkNr int) int64 { func parseFragmentedMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel int, parameterSets bool, nrRaw int) error { var trex *mp4.TrexBox + var avcSPS *avc.SPS + var err error if f.Init != nil { // Auto-detect codec if moov box is there moov := f.Init.Moov videoTrak, ok := findFirstVideoTrak(moov) if !ok { - return fmt.Errorf("No video track found") + return fmt.Errorf("no video track found") } stbl := videoTrak.Mdia.Minf.Stbl if stbl.Stsd.AvcX != nil { codec = "avc" + if stbl.Stsd.AvcX.AvcC != nil { + avcSPS, err = avc.ParseSPSNALUnit(stbl.Stsd.AvcX.AvcC.SPSnalus[0], true) + if err != nil { + return fmt.Errorf("error parsing SPS: %s", err) + } + } } else if stbl.Stsd.HvcX != nil { codec = "hevc" } @@ -215,11 +259,11 @@ func parseFragmentedMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel in } switch codec { case "avc", "h.264", "h264": - err = printAVCNalus(nalus, i+1, s.PresentationTime(), seiLevel, parameterSets, nrRaw) + err = printAVCNalus(avcSPS, nalus, i+1, s.PresentationTime(), seiLevel, parameterSets, nrRaw) case "hevc", "h.265", "h265": err = printHEVCNalus(nalus, i+1, s.PresentationTime(), seiLevel, parameterSets, nrRaw) default: - return fmt.Errorf("Unknown codec: %s", codec) + return fmt.Errorf("unknown codec: %s", codec) } if err != nil { @@ -232,11 +276,10 @@ func parseFragmentedMp4(f *mp4.File, maxNrSamples int, codec string, seiLevel in return nil } -func printAVCNalus(nalus [][]byte, nr int, pts uint64, seiLevel int, parameterSets bool, nrRaw int) error { +func printAVCNalus(avcSPS *avc.SPS, nalus [][]byte, nr int, pts uint64, seiLevel int, parameterSets bool, nrRaw int) error { msg := "" var seiNALUs [][]byte totLen := 0 - var avcSPS *avc.SPS for i, nalu := range nalus { totLen += 4 + len(nalu) if i > 0 { @@ -249,7 +292,7 @@ func printAVCNalus(nalus [][]byte, nr int, pts uint64, seiLevel int, parameterSe case avc.NALU_SPS: avcSPS, err = avc.ParseSPSNALUnit(nalu, true) if err != nil { - return fmt.Errorf("Error parsing SPS: %s", err) + return fmt.Errorf("error parsing SPS: %s", err) } case avc.NALU_NON_IDR, avc.NALU_IDR: sliceType, err := avc.GetSliceTypeFromNALU(nalu) @@ -389,3 +432,37 @@ func bytesToStringN(data []byte, maxNrBytes int) string { } return hex.EncodeToString(data) } + +func findAnnexBFrames(nalus [][]byte, codec string) ([][][]byte, error) { + var isAUD func([]byte) bool + switch codec { + case "avc": + isAUD = isAvcAudNalu + case "hevc": + isAUD = isHEVCAudNalu + default: + return nil, fmt.Errorf("unknown codec: %s", codec) + } + var frames [][][]byte + frameStart := 0 + for i, nalu := range nalus { + if isAUD(nalu) { + if i > frameStart { + frames = append(frames, nalus[frameStart:i]) + frameStart = i + } + } + } + if frameStart < len(nalus) { + frames = append(frames, nalus[frameStart:]) + } + return frames, nil +} + +func isAvcAudNalu(nalu []byte) bool { + return avc.GetNaluType(nalu[0]) == avc.NALU_AUD +} + +func isHEVCAudNalu(nalu []byte) bool { + return hevc.GetNaluType(nalu[0]) == hevc.NALU_AUD +} diff --git a/sei/sei1.go b/sei/sei1.go index 07245185..c78e4164 100644 --- a/sei/sei1.go +++ b/sei/sei1.go @@ -40,6 +40,7 @@ func DecodePicTimingAvcSEI(sd *SEIData) (SEIMessage, error) { // DecodePicTimingAvcSEIHRD decodes AVC SEI message 1 PicTiming with HRD parameters. // cbpDbpDelay length fields must be properly set if cbpDbpDelay is not nil. +// The delay values in cbpDbpDelay will then be set by the decoder by reading the bits. // It is assumed that pict_struct_present_flag is true, so that a 4-bit pict_struct value is present. func DecodePicTimingAvcSEIHRD(sd *SEIData, cbpDbpDelay *CbpDbpDelay, timeOffsetLen byte) (SEIMessage, error) { buf := bytes.NewBuffer(sd.Payload()) diff --git a/sei/sei_test.go b/sei/sei_test.go index 6135ce42..ec5ba625 100644 --- a/sei/sei_test.go +++ b/sei/sei_test.go @@ -282,38 +282,60 @@ const ( seiCEA608Hex = "0434b500314741393403cefffc9420fc94aefc9162fce56efc67bafc91b9" + "fcb0b0fcbab0fcb0bafcb031fcbab0fcb080fc942cfc942f80" seiAVCMulti = "0001c001061b0509b8000080" + seiAVCPicTiming = "010f00011a00000300090c2e268a000003004080" missingRbspTrailingBits = "01061b0509b80000" seiHEVCMulti = "000a8000000300403dc017a6900105040000be05880660404198b41080" seiHEVCHDR = "891800000300000300000300000300000300000300000300000300000300000300000300009004000003000080" ) +type avcHRD struct { + cbpDelay *sei.CbpDbpDelay + timeOffsetLen byte +} + func TestParseSEI(t *testing.T) { testCases := []struct { name string codec sei.Codec naluHex string + avcHRD *avcHRD wantedTypes []uint wantedStrings []string expNonFatalErr error }{ - {"AVC multi", sei.AVC, seiAVCMulti, []uint{0, 1}, + {"AVC PicTiming", sei.AVC, seiAVCPicTiming, + &avcHRD{ + cbpDelay: &sei.CbpDbpDelay{ + InitialCpbRemovalDelayLengthMinus1: 23, + CpbRemovalDelayLengthMinus1: 23, + DpbOutputDelayLengthMinus1: 23, + }, + timeOffsetLen: 24, + }, + []uint{1}, + []string{ + `SEIPicTimingType (1), size=15, time=20:40:09:46 offset=24`, + }, + nil, + }, + {"AVC multi", sei.AVC, seiAVCMulti, nil, []uint{0, 1}, []string{ `SEIBufferingPeriodType (0), size=1, "c0"`, `SEIPicTimingType (1), size=6, time=00:00:46:09 offset=0`, }, nil, }, - {"Missing RBSP Trailing Bits", sei.AVC, missingRbspTrailingBits, []uint{1}, + {"Missing RBSP Trailing Bits", sei.AVC, missingRbspTrailingBits, nil, []uint{1}, []string{ `SEIPicTimingType (1), size=6, time=00:00:46:09 offset=0`, }, sei.ErrRbspTrailingBitsMissing, }, - {"Type 0", sei.AVC, sei0Hex, []uint{0}, []string{`SEIBufferingPeriodType (0), size=7, "810f1c00507440"`}, nil}, - {"CEA-608", sei.AVC, seiCEA608Hex, []uint{4}, + {"Type 0", sei.AVC, sei0Hex, nil, []uint{0}, []string{`SEIBufferingPeriodType (0), size=7, "810f1c00507440"`}, nil}, + {"CEA-608", sei.AVC, seiCEA608Hex, nil, []uint{4}, []string{`SEI type 4 CEA-608, size=52, field1: "942094ae9162e56e67ba91b9b0b0bab0b0bab031bab0b080942c942f", field2: ""`}, nil}, - {"HEVC multi", sei.HEVC, seiHEVCMulti, []uint{0, 1, 136}, + {"HEVC multi", sei.HEVC, seiHEVCMulti, nil, []uint{0, 1, 136}, []string{ `SEIBufferingPeriodType (0), size=10, "80000000403dc017a690"`, `SEIPicTimingType (1), size=5, "040000be05"`, @@ -321,7 +343,7 @@ func TestParseSEI(t *testing.T) { }, nil, }, - {"Type HDR HEVC", sei.HEVC, seiHEVCHDR, []uint{137, 144}, + {"Type HDR HEVC", sei.HEVC, seiHEVCHDR, nil, []uint{137, 144}, []string{ "SEIMasteringDisplayColourVolumeType (137) 24B: primaries=(0, 0) (0, 0) (0, 0)," + " whitePoint=(0, 0), maxLum=0, minLum=0", @@ -344,7 +366,13 @@ func TestParseSEI(t *testing.T) { t.Errorf("%s: Not %d but %d sei messages found", tc.name, len(tc.wantedStrings), len(seis)) } for i := range seis { - seiMessage, err := sei.DecodeSEIMessage(&seis[i], tc.codec) + var seiMessage sei.SEIMessage + if tc.avcHRD == nil { + seiMessage, err = sei.DecodeSEIMessage(&seis[i], tc.codec) + } else { + seiMessage, err = sei.DecodePicTimingAvcSEIHRD(&seis[i], tc.avcHRD.cbpDelay, tc.avcHRD.timeOffsetLen) + + } if err != nil { t.Error(err) }