Gstreamer录制带音频的视频。

Question

Gstreamer录制带音频的视频。

4

我正在尝试使用Gstreamer通过glib库，在我的Ubuntu 16计算机上录制来自网络摄像头的视频和音频，并将其保存在文件中。
我能够通过以下代码行观看来自网络摄像头的视频流。

#include <gst/gst.h>

int main(int argc, char *argv[]) {
    GstElement *pipeline, *source, *sink, *convert;
    GstBus *bus;
    GstMessage *msg;
    GstStateChangeReturn ret;

    /* Initialize GStreamer */
    gst_init (&argc, &argv);

    /* Create the elements */
    source = gst_element_factory_make ("v4l2src", "source");
    sink = gst_element_factory_make ("autovideosink", "sink");
    convert =gst_element_factory_make("videoconvert","convert");
    //convert = gst_element_factory_make ("audioconvert", "convert");
    //sink = gst_element_factory_make ("autoaudiosink", "sink");

    /* Create the empty pipeline */
    pipeline = gst_pipeline_new ("test-pipeline");

    if (!pipeline || !source || !sink || !convert) {
        g_printerr ("Not all elements could be created.\n");
        return -1;
    }

    /*set der source*/
    g_object_set (source, "device", "/dev/video0", NULL);

    /* Build the pipeline */
    gst_bin_add_many (GST_BIN (pipeline), source, sink, convert, NULL);
    if (gst_element_link (convert, sink) != TRUE) {
        g_printerr ("Elements could not be linked confert sink.\n");
        gst_object_unref (pipeline);
        return -1;
    }


    if (gst_element_link (source, convert) != TRUE) {
        g_printerr ("Elements could not be linked source -convert.\n");
        gst_object_unref (pipeline);
        return -1;
    }

    /* Start playing */
    ret = gst_element_set_state (pipeline, GST_STATE_PLAYING);
    if (ret == GST_STATE_CHANGE_FAILURE) {
        g_printerr ("Unable to set the pipeline to the playing state.\n");
        gst_object_unref (pipeline);
        return -1;
    }

    /* Wait until error or EOS */
    bus = gst_element_get_bus (pipeline);
    msg = gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE,(GstMessageType) (GST_MESSAGE_ERROR | GST_MESSAGE_EOS));

    /* Parse message */
    if (msg != NULL) {
        GError *err;
        gchar *debug_info;

        switch (GST_MESSAGE_TYPE (msg)) {
            case GST_MESSAGE_ERROR:
                gst_message_parse_error (msg, &err, &debug_info);
                g_printerr ("Error received from element %s: %s\n", GST_OBJECT_NAME (msg->src), err->message);
                g_printerr ("Debugging information: %s\n", debug_info ? debug_info : "none");
                g_clear_error (&err);
                g_free (debug_info);
                break;
            case GST_MESSAGE_EOS:
                g_print ("End-Of-Stream reached.\n");
                break;
            default:
                /* We should not reach here because we only asked for ERRORs and EOS */
                g_printerr ("Unexpected message received.\n");
                break;
        }
        gst_message_unref (msg);
    }

    /* Free resources */
    gst_object_unref (bus);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    gst_object_unref (pipeline);
    return 0;
}

并使用下面这些代码行从麦克风捕获音频，并通过扬声器进行监听。

#include <gst/gst.h>
#include <glib.h>

static gboolean
bus_call (GstBus     *bus,
          GstMessage *msg,
          gpointer    data){
  GMainLoop *loop = (GMainLoop *) data;

  switch (GST_MESSAGE_TYPE (msg)) {

    case GST_MESSAGE_EOS:
      g_print ("End of stream\n");
      g_main_loop_quit (loop);
      break;

    case GST_MESSAGE_ERROR: {
      gchar  *debug;
      GError *error;

      gst_message_parse_error (msg, &error, &debug);
      g_free (debug);

      g_printerr ("Error: %s\n", error->message);
      g_error_free (error);

      g_main_loop_quit (loop);
      break;
    }
    default:
      break;
  }

  return TRUE;
}

/* Main function for audio pipeline initialization and looping streaming process  */
gint
main (gint argc, gchar **argv) {
    GMainLoop *loop;
    GstElement *pipeline, *audio_source, *sink; 
    GstBus *bus;
    guint bus_watch_id;
    GstCaps *caps;
    gboolean ret;

    /* Initialization of gstreamer */
    gst_init (&argc, &argv);
    loop = g_main_loop_new (NULL, FALSE);

    /* Elements creation */
    pipeline     = gst_pipeline_new ("audio_stream");
    audio_source = gst_element_factory_make ("alsasrc", "audio_source");
    sink   = gst_element_factory_make ("alsasink", "audio_sink");

    // video_source = gst_element_factory_make ("v4l2src", "source");
    // video_sink   = gst_element_factory_make ("autovideosink", "sink");
    // video_convert= gst_element_factory_make("videoconvert","convert");

    if (!pipeline) {
        g_printerr ("Audio: Pipeline couldn't be created\n");
        return -1;
    }
    if (!audio_source) {
        g_printerr ("Audio: alsasrc couldn't be created\n");
        return -1;
    }
    if (!sink) {
        g_printerr ("Audio: Output file couldn't be created\n");
        return -1;
    }

    g_object_set (G_OBJECT (audio_source), "device", "hw:1,0", NULL);
    g_object_set (G_OBJECT (sink), "device", "hw:1,0", NULL);

    bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
    bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
    gst_object_unref (bus);

    gst_bin_add_many (GST_BIN(pipeline), audio_source, sink, NULL);

    caps = gst_caps_new_simple ("audio/x-raw", "format", G_TYPE_STRING, "S16LE",  "layout", G_TYPE_STRING, "interleaved", "rate", G_TYPE_INT, (int)44100, "channels", G_TYPE_INT, (int)2, NULL);
    ret = gst_element_link_filtered (audio_source, sink, caps);
    if (!ret) {
        g_print ("audio_source and sink couldn't be linked\n");
        gst_caps_unref (caps);
        return FALSE;
    }

    gst_element_set_state (pipeline, GST_STATE_PLAYING);

    g_print ("streaming...\n");
    g_main_loop_run (loop);

    g_print ("Returned, stopping stream\n");
    gst_element_set_state (pipeline, GST_STATE_NULL);

    g_print ("Deleting pipeline\n");
    gst_object_unref (GST_OBJECT (pipeline));
    g_source_remove (bus_watch_id);
    g_main_loop_unref (loop);

    return 0;
}

我真正不理解的是如何同时从网络摄像头获取视频和从我的alsa hw获取音频，并将它们保存到文件中（例如.mp4）。有人能帮我吗？我尝试找到一些有用的东西，但是论坛上没有。此外，如果能够将仅视频流或仅音频流保存在单独的文件中，那将非常感激。

更新我再次查看了教程和@nayana提供的git链接，因此我尝试自己编写了一些代码。我有两个结果：

#include <string.h>
#include <gst/gst.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>

static GMainLoop *loop;
static GstElement *pipeline;
static GstElement *muxer, *sink;
static GstElement *src_video, *encoder_video, *queue_video; 
static GstElement *src_audio, *encoder_audio, *queue_audio;
static GstBus *bus;

static gboolean
message_cb (GstBus * bus, GstMessage * message, gpointer user_data)
{
  switch (GST_MESSAGE_TYPE (message)) {
    case GST_MESSAGE_ERROR:{
      GError *err = NULL;
      gchar *name, *debug = NULL;

      name = gst_object_get_path_string (message->src);
      gst_message_parse_error (message, &err, &debug);

      g_printerr ("ERROR: from element %s: %s\n", name, err->message);
      if (debug != NULL)
        g_printerr ("Additional debug info:\n%s\n", debug);

      g_error_free (err);
      g_free (debug);
      g_free (name);

      g_main_loop_quit (loop);
      break;
    }
    case GST_MESSAGE_WARNING:{
    GError *err = NULL;
    gchar *name, *debug = NULL;

    name = gst_object_get_path_string (message->src);
    gst_message_parse_warning (message, &err, &debug);

    g_printerr ("ERROR: from element %s: %s\n", name, err->message);
    if (debug != NULL)
    g_printerr ("Additional debug info:\n%s\n", debug);

    g_error_free (err);
    g_free (debug);
    g_free (name);
    break;
    }
    case GST_MESSAGE_EOS:{
    g_print ("Got EOS\n");
    g_main_loop_quit (loop);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    g_main_loop_unref (loop);
    gst_object_unref (pipeline);
    exit(0);
    break;
  }
    default:
    break;
  }

  return TRUE;
}

void sigintHandler(int unused) {
  g_print("You ctrl-c-ed! Sending EoS");
  gst_element_send_event(pipeline, gst_event_new_eos()); 
}

int main(int argc, char *argv[])
{
  signal(SIGINT, sigintHandler);
  gst_init (&argc, &argv);

  pipeline = gst_pipeline_new(NULL);

  src_video = gst_element_factory_make("v4l2src", NULL);
  encoder_video = gst_element_factory_make("x264enc", NULL);
  queue_video = gst_element_factory_make("queue", NULL);

  src_audio = gst_element_factory_make ("alsasrc", NULL);
  encoder_audio = gst_element_factory_make("lamemp3enc", NULL);
  queue_audio = gst_element_factory_make("queue", NULL);

  muxer = gst_element_factory_make("mp4mux", NULL);
  sink = gst_element_factory_make("filesink", NULL);

  if (!pipeline || !src_video || !encoder_video || !src_audio || !encoder_audio
        || !queue_video || !queue_audio || !muxer || !sink) {
    g_error("Failed to create elements");
    return -1;
  }

  g_object_set(src_audio, "device", "hw:1,0", NULL);
  g_object_set(sink, "location", "video_audio_test.mp4", NULL);


  gst_bin_add_many(GST_BIN(pipeline), src_video, encoder_video, queue_video, 
    src_audio, encoder_audio, queue_audio, muxer, sink, NULL);

  gst_element_link_many (src_video,encoder_video,queue_video, muxer,NULL);

  gst_element_link_many (src_audio,encoder_audio,queue_audio, muxer,NULL);

  if (!gst_element_link_many(muxer, sink, NULL)){
    g_error("Failed to link elements");
    return -2;
  }

  loop = g_main_loop_new(NULL, FALSE);

  bus = gst_pipeline_get_bus(GST_PIPELINE (pipeline));
  gst_bus_add_signal_watch(bus);
  g_signal_connect(G_OBJECT(bus), "message", G_CALLBACK(message_cb), NULL);
  gst_object_unref(GST_OBJECT(bus));

  gst_element_set_state(pipeline, GST_STATE_PLAYING);

  g_print("Starting loop");
  g_main_loop_run(loop);

  return 0;
}

通过这种方式，我能够录制来自摄像头的视频，但是在录制过程中，音频仅随机地记录了一秒钟，并给出了以下错误提示：

ERROR: from element /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0: Can't record audio fast enough
Additional debug info:
gstaudiobasesrc.c(869): gst_audio_base_src_create (): /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0:
Dropped 206388 samples. This is most likely because downstream can't keep up and is consuming samples too slowly.<br>

所以我尝试添加一些设置和队列

#include <string.h>
#include <gst/gst.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>

static GMainLoop *loop;
static GstElement *pipeline;
static GstElement *muxer, *sink;
static GstElement *src_video, *encoder_video, *queue_video, *rate_video, *scale_video, *capsfilter_video; 
static GstElement *src_audio, *encoder_audio, *queue_audio, *queue_audio2, *capsfilter_audio, *rate_audio;
static GstBus *bus;
static GstCaps *caps;

static gboolean
message_cb (GstBus * bus, GstMessage * message, gpointer user_data)
{
  switch (GST_MESSAGE_TYPE (message)) {
    case GST_MESSAGE_ERROR:{
      GError *err = NULL;
      gchar *name, *debug = NULL;

      name = gst_object_get_path_string (message->src);
      gst_message_parse_error (message, &err, &debug);

      g_printerr ("ERROR: from element %s: %s\n", name, err->message);
      if (debug != NULL)
        g_printerr ("Additional debug info:\n%s\n", debug);

      g_error_free (err);
      g_free (debug);
      g_free (name);

      g_main_loop_quit (loop);
      break;
    }
    case GST_MESSAGE_WARNING:{
    GError *err = NULL;
    gchar *name, *debug = NULL;

    name = gst_object_get_path_string (message->src);
    gst_message_parse_warning (message, &err, &debug);

    g_printerr ("ERROR: from element %s: %s\n", name, err->message);
    if (debug != NULL)
    g_printerr ("Additional debug info:\n%s\n", debug);

    g_error_free (err);
    g_free (debug);
    g_free (name);
    break;
    }
    case GST_MESSAGE_EOS:{
    g_print ("Got EOS\n");
    g_main_loop_quit (loop);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    g_main_loop_unref (loop);
    gst_object_unref (pipeline);
    exit(0);
    break;
  }
    default:
    break;
  }

  return TRUE;
}

void sigintHandler(int unused) {
  g_print("You ctrl-c-ed! Sending EoS");
  gst_element_send_event(pipeline, gst_event_new_eos()); 
}

int main(int argc, char *argv[])
{
  signal(SIGINT, sigintHandler);
  gst_init (&argc, &argv);

  pipeline = gst_pipeline_new(NULL);

  src_video = gst_element_factory_make("v4l2src", NULL);
  rate_video = gst_element_factory_make ("videorate", NULL);
  scale_video = gst_element_factory_make ("videoscale", NULL);
  capsfilter_video = gst_element_factory_make ("capsfilter", NULL);
  queue_video = gst_element_factory_make("queue", NULL);
  encoder_video = gst_element_factory_make("x264enc", NULL);

  src_audio = gst_element_factory_make ("alsasrc", NULL);
  capsfilter_audio = gst_element_factory_make ("capsfilter", NULL);
  queue_audio = gst_element_factory_make("queue", NULL);
  rate_audio = gst_element_factory_make ("audiorate", NULL);
  queue_audio2 = gst_element_factory_make("queue", NULL);
  encoder_audio = gst_element_factory_make("lamemp3enc", NULL);

  muxer = gst_element_factory_make("mp4mux", NULL);
  sink = gst_element_factory_make("filesink", NULL);

  if (!pipeline || !src_video || !rate_video || !scale_video || !capsfilter_video 
     || !queue_video || !encoder_video || !src_audio || !capsfilter_audio 
     || !queue_audio || !rate_audio || !queue_audio2 || !encoder_audio 
     || !muxer || !sink) {
    g_error("Failed to create elements");
    return -1;
  }

  // Set up the pipeline
  g_object_set(src_video, "device", "/dev/video0", NULL); 
  g_object_set(src_audio, "device", "hw:1,0", NULL);
  g_object_set(sink, "location", "video_audio_test.mp4", NULL);

  // video settings
  caps = gst_caps_from_string("video/x-raw,format=(string)I420,width=480,height=384,framerate=(fraction)25/1");
  g_object_set (G_OBJECT (capsfilter_video), "caps", caps, NULL);
  gst_caps_unref (caps); 

  // audio settings
  caps = gst_caps_from_string("audio/x-raw,rate=44100,channels=1");
  g_object_set (G_OBJECT (capsfilter_audio), "caps", caps, NULL);
  gst_caps_unref (caps);

  // add all elements into the pipeline 
  gst_bin_add_many(GST_BIN(pipeline), src_video, rate_video, scale_video, capsfilter_video, 
    queue_video, encoder_video, src_audio, capsfilter_audio, queue_audio, rate_audio, 
    queue_audio2, encoder_audio, muxer, sink, NULL);

  if (!gst_element_link_many (src_video,rate_video,scale_video, capsfilter_video,
    queue_video, encoder_video, muxer,NULL))
  {
    g_error("Failed to link video elements");
    return -2;
  }

  if (!gst_element_link_many (src_audio, capsfilter_audio, queue_audio, rate_audio, 
    queue_audio2, encoder_audio, muxer,NULL))
  {
    g_error("Failed to link audio elements");
    return -2;
  }

  if (!gst_element_link_many(muxer, sink, NULL))
  {
    g_error("Failed to link elements");
    return -2;
  }

  loop = g_main_loop_new(NULL, FALSE);

  bus = gst_pipeline_get_bus(GST_PIPELINE (pipeline));
  gst_bus_add_signal_watch(bus);
  g_signal_connect(G_OBJECT(bus), "message", G_CALLBACK(message_cb), NULL);
  gst_object_unref(GST_OBJECT(bus));

  gst_element_set_state(pipeline, GST_STATE_PLAYING);

  g_print("Starting loop");
  g_main_loop_run(loop);

  return 0;
}

这次代码没有记录任何内容，并给出以下错误：

   ERROR: from element /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0: Internal data flow error.
Additional debug info:
gstbasesrc.c(2948): gst_base_src_loop (): /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0:
streaming task paused, reason not-negotiated (-4)

你能帮我解决这个错误吗？
提前感谢。

- Antmau

好的，很不错。尝试在第一个解决方案中为alsasrc添加“do-timestamp=true”。此外，您可以通过使用环境变量“GST_DEBUG=5 ./your_binary”运行二进制文件来调试正在进行的操作。另一件事是尝试使用qtmux，它比mp4mux具有一些快速时间扩展（如果我没记错的话）。 - nayana

据我所了解，需要对视频流进行某种转换，以减少处理量并将其与音频流同步。我仍然面临这个问题，因为我无法为源设置正确的管道和 caps 过滤器（我想要降低分辨率和帧速率）。 - Antmau

我发现了一篇非常有趣的帖子，建议使用zerolatency，这可以解决x264enc无法跟上lamemp3enc的问题。http://gstreamer-devel.966125.n4.nabble.com/audio-video-out-of-sync-td4675526.html 在第一个解决方案中，在编码器之前添加队列。第二个解决方案过于复杂，我不太理解。再次强调，先用gst-launch尝试一下。 - nayana

1个回答

网页内容由stack overflow 提供, 点击上面的

可以查看英文原文，
原文链接

- nayana · Accepted Answer

你需要的是复用器——这样的GStreamer元素可以将两个流合并为一个。

mp4、mkv、avi等只是包含多个“数据流”的容器格式，其中可以是音频、视频、字幕（并非所有格式都支持此项）。

我不知道你的使用情况，但你不需要C代码来完成你的任务。你可以使用gst-launch-1.0工具，它有自己的GStreamer脚本语言。

为了简单起见，我将使用调试元素videotestsrc和audiotestsrc来模拟输入（而不是实际相机等）。

gst-launch-1.0 -e videotestsrc ! x264enc ! mp4mux name=mux ! filesink location="bla.mp4"  audiotestsrc ! lamemp3enc ! mux.

videotestsrc --> x264enc -----\
                               >---> mp4mux ---> filesink
audiotestsrc --> lamemp3enc --/

解释：

Videotestsrc生成的是GStreamer术语中称为“video/x-raw”的原始视频。

然而，mp4无法保存原始视频，因此我们需要使用x264enc进行编码，使我们的数据变成“video/x-h264”格式。

然后我们可以最终使用mp4mux元素将其混合到我们的mp4中。

当我们查看GStreamer文档并使用gst-inspect-1.0 mp4mux时，我们可以看到该元素支持各种格式，其中还包括video/x-h264。

我们使用faac或lamemp3enc来处理音频。

使用gst-launch-1.0时，我做了两个技巧和一个额外的技巧：

能够在一行中有单独的分支。只需使用空格而不是!来分隔这些分支即可实现。
能够使用name=mux创建别名，并在名称末尾添加点号（如mux.）来稍后使用它。您可以为该元素制定任何名称。
按下ctrl+c停止录制后，写入EOS。这通过参数-e实现。

最后输出到filesink，它只是将任何给定的内容写入文件中。

现在轮到你做作业了：

使用您需要的元素-v4l2，alsasrc
添加队列元素以增加缓冲区和线程分离