Gstreamer 录制带音频的视频 [英] Gstreamer recording video with audio

查看：112
本文介绍了Gstreamer 录制带音频的视频的处理方法，对大家解决问题具有一定的参考价值，需要的朋友们下面随着小编来一起学习吧！

问题描述


我正在尝试通过 glib 库在我的 Ubuntu 16 机器上使用 Gstreamer 将来自网络摄像头的视频和音频录制到文件中.
我可以通过这些代码行观看来自网络摄像头的视频流


I'm trying to record on a file a video from my webcam along with audio using Gstreamer on my Ubuntu 16 machine through glib library.
I'm able to watch the video streaming from the webcam through these code lines

#include <gst/gst.h>

int main(int argc, char *argv[]) {
    GstElement *pipeline, *source, *sink, *convert;
    GstBus *bus;
    GstMessage *msg;
    GstStateChangeReturn ret;

    /* Initialize GStreamer */
    gst_init (&argc, &argv);

    /* Create the elements */
    source = gst_element_factory_make ("v4l2src", "source");
    sink = gst_element_factory_make ("autovideosink", "sink");
    convert =gst_element_factory_make("videoconvert","convert");
    //convert = gst_element_factory_make ("audioconvert", "convert");
    //sink = gst_element_factory_make ("autoaudiosink", "sink");

    /* Create the empty pipeline */
    pipeline = gst_pipeline_new ("test-pipeline");

    if (!pipeline || !source || !sink || !convert) {
        g_printerr ("Not all elements could be created.\n");
        return -1;
    }

    /*set der source*/
    g_object_set (source, "device", "/dev/video0", NULL);

    /* Build the pipeline */
    gst_bin_add_many (GST_BIN (pipeline), source, sink, convert, NULL);
    if (gst_element_link (convert, sink) != TRUE) {
        g_printerr ("Elements could not be linked confert sink.\n");
        gst_object_unref (pipeline);
        return -1;
    }


    if (gst_element_link (source, convert) != TRUE) {
        g_printerr ("Elements could not be linked source -convert.\n");
        gst_object_unref (pipeline);
        return -1;
    }

    /* Start playing */
    ret = gst_element_set_state (pipeline, GST_STATE_PLAYING);
    if (ret == GST_STATE_CHANGE_FAILURE) {
        g_printerr ("Unable to set the pipeline to the playing state.\n");
        gst_object_unref (pipeline);
        return -1;
    }

    /* Wait until error or EOS */
    bus = gst_element_get_bus (pipeline);
    msg = gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE,(GstMessageType) (GST_MESSAGE_ERROR | GST_MESSAGE_EOS));

    /* Parse message */
    if (msg != NULL) {
        GError *err;
        gchar *debug_info;

        switch (GST_MESSAGE_TYPE (msg)) {
            case GST_MESSAGE_ERROR:
                gst_message_parse_error (msg, &err, &debug_info);
                g_printerr ("Error received from element %s: %s\n", GST_OBJECT_NAME (msg->src), err->message);
                g_printerr ("Debugging information: %s\n", debug_info ? debug_info : "none");
                g_clear_error (&err);
                g_free (debug_info);
                break;
            case GST_MESSAGE_EOS:
                g_print ("End-Of-Stream reached.\n");
                break;
            default:
                /* We should not reach here because we only asked for ERRORs and EOS */
                g_printerr ("Unexpected message received.\n");
                break;
        }
        gst_message_unref (msg);
    }

    /* Free resources */
    gst_object_unref (bus);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    gst_object_unref (pipeline);
    return 0;
}


并使用这些代码行从麦克风捕获音频并通过扬声器收听它


and to capture audio from microphone and listen it through the speakers using these code lines

#include <gst/gst.h>
#include <glib.h>

static gboolean
bus_call (GstBus     *bus,
          GstMessage *msg,
          gpointer    data){
  GMainLoop *loop = (GMainLoop *) data;

  switch (GST_MESSAGE_TYPE (msg)) {

    case GST_MESSAGE_EOS:
      g_print ("End of stream\n");
      g_main_loop_quit (loop);
      break;

    case GST_MESSAGE_ERROR: {
      gchar  *debug;
      GError *error;

      gst_message_parse_error (msg, &error, &debug);
      g_free (debug);

      g_printerr ("Error: %s\n", error->message);
      g_error_free (error);

      g_main_loop_quit (loop);
      break;
    }
    default:
      break;
  }

  return TRUE;
}

/* Main function for audio pipeline initialization and looping streaming process  */
gint
main (gint argc, gchar **argv) {
    GMainLoop *loop;
    GstElement *pipeline, *audio_source, *sink; 
    GstBus *bus;
    guint bus_watch_id;
    GstCaps *caps;
    gboolean ret;

    /* Initialization of gstreamer */
    gst_init (&argc, &argv);
    loop = g_main_loop_new (NULL, FALSE);

    /* Elements creation */
    pipeline     = gst_pipeline_new ("audio_stream");
    audio_source = gst_element_factory_make ("alsasrc", "audio_source");
    sink   = gst_element_factory_make ("alsasink", "audio_sink");

    // video_source = gst_element_factory_make ("v4l2src", "source");
    // video_sink   = gst_element_factory_make ("autovideosink", "sink");
    // video_convert= gst_element_factory_make("videoconvert","convert");

    if (!pipeline) {
        g_printerr ("Audio: Pipeline couldn't be created\n");
        return -1;
    }
    if (!audio_source) {
        g_printerr ("Audio: alsasrc couldn't be created\n");
        return -1;
    }
    if (!sink) {
        g_printerr ("Audio: Output file couldn't be created\n");
        return -1;
    }

    g_object_set (G_OBJECT (audio_source), "device", "hw:1,0", NULL);
    g_object_set (G_OBJECT (sink), "device", "hw:1,0", NULL);

    bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
    bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
    gst_object_unref (bus);

    gst_bin_add_many (GST_BIN(pipeline), audio_source, sink, NULL);

    caps = gst_caps_new_simple ("audio/x-raw", "format", G_TYPE_STRING, "S16LE",  "layout", G_TYPE_STRING, "interleaved", "rate", G_TYPE_INT, (int)44100, "channels", G_TYPE_INT, (int)2, NULL);
    ret = gst_element_link_filtered (audio_source, sink, caps);
    if (!ret) {
        g_print ("audio_source and sink couldn't be linked\n");
        gst_caps_unref (caps);
        return FALSE;
    }

    gst_element_set_state (pipeline, GST_STATE_PLAYING);

    g_print ("streaming...\n");
    g_main_loop_run (loop);

    g_print ("Returned, stopping stream\n");
    gst_element_set_state (pipeline, GST_STATE_NULL);

    g_print ("Deleting pipeline\n");
    gst_object_unref (GST_OBJECT (pipeline));
    g_source_remove (bus_watch_id);
    g_main_loop_unref (loop);

    return 0;
}


我真正不明白的是如何同时从网络摄像头获取视频和从我的 alsa hw 获取音频并将它们保存到文件中(例如 .mp4 文件).谁能帮我?我试图找到一些有用的东西，但板上什么也没有.此外，如何将视频流或音频流保存在单独的文件中也将非常感激.


What i really don't understand is how to get video from the webcam and audio from my alsa hw at the same time and save them into a file (such as .mp4 for ex). Can anyone help me? I tried to find something useful, but there's nothing on the board. In addition, it would be really appreciate also how to save just the video stream or just the audio stream in separated files.

更新
我再次查看教程和@nayana 提供的 git 链接，所以我尝试自己编写代码.我有两个结果:

UPDATE
I looked again to the tutorials and to the git link gave by @nayana, so i tried myself to code something. I have two results: 

#include <string.h>
#include <gst/gst.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>

static GMainLoop *loop;
static GstElement *pipeline;
static GstElement *muxer, *sink;
static GstElement *src_video, *encoder_video, *queue_video; 
static GstElement *src_audio, *encoder_audio, *queue_audio;
static GstBus *bus;

static gboolean
message_cb (GstBus * bus, GstMessage * message, gpointer user_data)
{
  switch (GST_MESSAGE_TYPE (message)) {
    case GST_MESSAGE_ERROR:{
      GError *err = NULL;
      gchar *name, *debug = NULL;

      name = gst_object_get_path_string (message->src);
      gst_message_parse_error (message, &err, &debug);

      g_printerr ("ERROR: from element %s: %s\n", name, err->message);
      if (debug != NULL)
        g_printerr ("Additional debug info:\n%s\n", debug);

      g_error_free (err);
      g_free (debug);
      g_free (name);

      g_main_loop_quit (loop);
      break;
    }
    case GST_MESSAGE_WARNING:{
    GError *err = NULL;
    gchar *name, *debug = NULL;

    name = gst_object_get_path_string (message->src);
    gst_message_parse_warning (message, &err, &debug);

    g_printerr ("ERROR: from element %s: %s\n", name, err->message);
    if (debug != NULL)
    g_printerr ("Additional debug info:\n%s\n", debug);

    g_error_free (err);
    g_free (debug);
    g_free (name);
    break;
    }
    case GST_MESSAGE_EOS:{
    g_print ("Got EOS\n");
    g_main_loop_quit (loop);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    g_main_loop_unref (loop);
    gst_object_unref (pipeline);
    exit(0);
    break;
  }
    default:
    break;
  }

  return TRUE;
}

void sigintHandler(int unused) {
  g_print("You ctrl-c-ed! Sending EoS");
  gst_element_send_event(pipeline, gst_event_new_eos()); 
}

int main(int argc, char *argv[])
{
  signal(SIGINT, sigintHandler);
  gst_init (&argc, &argv);

  pipeline = gst_pipeline_new(NULL);

  src_video = gst_element_factory_make("v4l2src", NULL);
  encoder_video = gst_element_factory_make("x264enc", NULL);
  queue_video = gst_element_factory_make("queue", NULL);

  src_audio = gst_element_factory_make ("alsasrc", NULL);
  encoder_audio = gst_element_factory_make("lamemp3enc", NULL);
  queue_audio = gst_element_factory_make("queue", NULL);

  muxer = gst_element_factory_make("mp4mux", NULL);
  sink = gst_element_factory_make("filesink", NULL);

  if (!pipeline || !src_video || !encoder_video || !src_audio || !encoder_audio
        || !queue_video || !queue_audio || !muxer || !sink) {
    g_error("Failed to create elements");
    return -1;
  }

  g_object_set(src_audio, "device", "hw:1,0", NULL);
  g_object_set(sink, "location", "video_audio_test.mp4", NULL);


  gst_bin_add_many(GST_BIN(pipeline), src_video, encoder_video, queue_video, 
    src_audio, encoder_audio, queue_audio, muxer, sink, NULL);

  gst_element_link_many (src_video,encoder_video,queue_video, muxer,NULL);

  gst_element_link_many (src_audio,encoder_audio,queue_audio, muxer,NULL);

  if (!gst_element_link_many(muxer, sink, NULL)){
    g_error("Failed to link elements");
    return -2;
  }

  loop = g_main_loop_new(NULL, FALSE);

  bus = gst_pipeline_get_bus(GST_PIPELINE (pipeline));
  gst_bus_add_signal_watch(bus);
  g_signal_connect(G_OBJECT(bus), "message", G_CALLBACK(message_cb), NULL);
  gst_object_unref(GST_OBJECT(bus));

  gst_element_set_state(pipeline, GST_STATE_PLAYING);

  g_print("Starting loop");
  g_main_loop_run(loop);

  return 0;
}

有了这个，我就可以从摄像头录制视频，但是在录制过程中，音频只在某处随机录制了一秒钟，这给了我这个错误

With this upon i am able to record the video from the cam, but the audio is recorded for just one second somewhere randomly during the recording and it gives me this error 

ERROR: from element /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0: Can't record audio fast enough
Additional debug info:
gstaudiobasesrc.c(869): gst_audio_base_src_create (): /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0:
Dropped 206388 samples. This is most likely because downstream can't keep up and is consuming samples too slowly.<br>

所以我尝试添加一些设置和队列

So i tried to add some setting and queues 

#include <string.h>
#include <gst/gst.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>

static GMainLoop *loop;
static GstElement *pipeline;
static GstElement *muxer, *sink;
static GstElement *src_video, *encoder_video, *queue_video, *rate_video, *scale_video, *capsfilter_video; 
static GstElement *src_audio, *encoder_audio, *queue_audio, *queue_audio2, *capsfilter_audio, *rate_audio;
static GstBus *bus;
static GstCaps *caps;

static gboolean
message_cb (GstBus * bus, GstMessage * message, gpointer user_data)
{
  switch (GST_MESSAGE_TYPE (message)) {
    case GST_MESSAGE_ERROR:{
      GError *err = NULL;
      gchar *name, *debug = NULL;

      name = gst_object_get_path_string (message->src);
      gst_message_parse_error (message, &err, &debug);

      g_printerr ("ERROR: from element %s: %s\n", name, err->message);
      if (debug != NULL)
        g_printerr ("Additional debug info:\n%s\n", debug);

      g_error_free (err);
      g_free (debug);
      g_free (name);

      g_main_loop_quit (loop);
      break;
    }
    case GST_MESSAGE_WARNING:{
    GError *err = NULL;
    gchar *name, *debug = NULL;

    name = gst_object_get_path_string (message->src);
    gst_message_parse_warning (message, &err, &debug);

    g_printerr ("ERROR: from element %s: %s\n", name, err->message);
    if (debug != NULL)
    g_printerr ("Additional debug info:\n%s\n", debug);

    g_error_free (err);
    g_free (debug);
    g_free (name);
    break;
    }
    case GST_MESSAGE_EOS:{
    g_print ("Got EOS\n");
    g_main_loop_quit (loop);
    gst_element_set_state (pipeline, GST_STATE_NULL);
    g_main_loop_unref (loop);
    gst_object_unref (pipeline);
    exit(0);
    break;
  }
    default:
    break;
  }

  return TRUE;
}

void sigintHandler(int unused) {
  g_print("You ctrl-c-ed! Sending EoS");
  gst_element_send_event(pipeline, gst_event_new_eos()); 
}

int main(int argc, char *argv[])
{
  signal(SIGINT, sigintHandler);
  gst_init (&argc, &argv);

  pipeline = gst_pipeline_new(NULL);

  src_video = gst_element_factory_make("v4l2src", NULL);
  rate_video = gst_element_factory_make ("videorate", NULL);
  scale_video = gst_element_factory_make ("videoscale", NULL);
  capsfilter_video = gst_element_factory_make ("capsfilter", NULL);
  queue_video = gst_element_factory_make("queue", NULL);
  encoder_video = gst_element_factory_make("x264enc", NULL);

  src_audio = gst_element_factory_make ("alsasrc", NULL);
  capsfilter_audio = gst_element_factory_make ("capsfilter", NULL);
  queue_audio = gst_element_factory_make("queue", NULL);
  rate_audio = gst_element_factory_make ("audiorate", NULL);
  queue_audio2 = gst_element_factory_make("queue", NULL);
  encoder_audio = gst_element_factory_make("lamemp3enc", NULL);

  muxer = gst_element_factory_make("mp4mux", NULL);
  sink = gst_element_factory_make("filesink", NULL);

  if (!pipeline || !src_video || !rate_video || !scale_video || !capsfilter_video 
     || !queue_video || !encoder_video || !src_audio || !capsfilter_audio 
     || !queue_audio || !rate_audio || !queue_audio2 || !encoder_audio 
     || !muxer || !sink) {
    g_error("Failed to create elements");
    return -1;
  }

  // Set up the pipeline
  g_object_set(src_video, "device", "/dev/video0", NULL); 
  g_object_set(src_audio, "device", "hw:1,0", NULL);
  g_object_set(sink, "location", "video_audio_test.mp4", NULL);

  // video settings
  caps = gst_caps_from_string("video/x-raw,format=(string)I420,width=480,height=384,framerate=(fraction)25/1");
  g_object_set (G_OBJECT (capsfilter_video), "caps", caps, NULL);
  gst_caps_unref (caps); 

  // audio settings
  caps = gst_caps_from_string("audio/x-raw,rate=44100,channels=1");
  g_object_set (G_OBJECT (capsfilter_audio), "caps", caps, NULL);
  gst_caps_unref (caps);

  // add all elements into the pipeline 
  gst_bin_add_many(GST_BIN(pipeline), src_video, rate_video, scale_video, capsfilter_video, 
    queue_video, encoder_video, src_audio, capsfilter_audio, queue_audio, rate_audio, 
    queue_audio2, encoder_audio, muxer, sink, NULL);

  if (!gst_element_link_many (src_video,rate_video,scale_video, capsfilter_video,
    queue_video, encoder_video, muxer,NULL))
  {
    g_error("Failed to link video elements");
    return -2;
  }

  if (!gst_element_link_many (src_audio, capsfilter_audio, queue_audio, rate_audio, 
    queue_audio2, encoder_audio, muxer,NULL))
  {
    g_error("Failed to link audio elements");
    return -2;
  }

  if (!gst_element_link_many(muxer, sink, NULL))
  {
    g_error("Failed to link elements");
    return -2;
  }

  loop = g_main_loop_new(NULL, FALSE);

  bus = gst_pipeline_get_bus(GST_PIPELINE (pipeline));
  gst_bus_add_signal_watch(bus);
  g_signal_connect(G_OBJECT(bus), "message", G_CALLBACK(message_cb), NULL);
  gst_object_unref(GST_OBJECT(bus));

  gst_element_set_state(pipeline, GST_STATE_PLAYING);

  g_print("Starting loop");
  g_main_loop_run(loop);

  return 0;
}

这次代码没有记录任何东西并给我以下错误

This time the code doesnt record anything and give me the following error

   ERROR: from element /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0: Internal data flow error.
Additional debug info:
gstbasesrc.c(2948): gst_base_src_loop (): /GstPipeline:pipeline0/GstAlsaSrc:alsasrc0:
streaming task paused, reason not-negotiated (-4)

你能解决我的错误吗?
提前致谢

Can you address me to fix the error?
Thanks in advance

推荐答案

您需要的是多路复用器——例如可以将两个流合并为一个的 GStreamer 元素.

What you need is the multiplexer - such GStreamer element that can merge two streams into one.

mp4、mkv、avi .. 只是一种包含多个数据流"的容器格式，可以是音频、视频、字幕(并非所有格式都支持).

mp4, mkv, avi.. are just a container formats which contains multiple "data streams", which can be audio, video, subtitles (not all formats support this).

我不知道您的用例，但是您的工作不需要 C 代码.你可以使用 gst-launch-1.0 工具，它有自己的 GStreamer 类脚本语言.

I don't know about your use case, but you don't need C code for what you do. You can just use gst-launch-1.0 tool which has its own GStreamer kind-of-scripting language.

为简单起见，我将使用调试元素 videotestsrc 和 audiotestsrc 来模拟输入(而不是实际的相机等).

For simplicity I will use debugging elements videotestsrc and audiotestsrc for simulating input (instead of actual camera etc).

gst-launch-1.0 -e videotestsrc ! x264enc ! mp4mux name=mux ! filesink location="bla.mp4"  audiotestsrc ! lamemp3enc ! mux.

videotestsrc --> x264enc -----\
                               >---> mp4mux ---> filesink
audiotestsrc --> lamemp3enc --/

说明:

Videotestsrc 生成原始视频，在 GStreamer 中称为video/x-raw".

Videotestsrc generates raw video which is in GStreamer terms called "video/x-raw".

然而 mp4 不能保存原始视频，所以我们需要使用例如 x264enc 对它进行编码，这使我们的数据视频/x-h264".

However mp4 cannot hold raw video, so we need to encode it with for example x264enc which makes our data "video/x-h264".

然后我们最终可以使用 mp4mux 元素将其混合到我们的 mp4 中.

Then we can finally mux this into our mp4 with mp4mux element.

当我们使用 gst-inspect-1.0 mp4mux 查看 GStreamer 文档时，我们看到该元素支持各种格式，其中还有 video/x-h264.

When we take a look into GStreamer docs using gst-inspect-1.0 mp4mux we see that this element supports various formats amongst which there is also video/x-h264.

我们对 AAC 格式的 faac 或 mp3 格式的 lamemmp3enc 音频所做的同样的事情.

The same thing we do with audio with either faac for AAC format or lamemp3enc for mp3.

使用 gst-launch-1.0 我做了两个技巧和一个奖励技巧:

With gst-launch-1.0 I did two tricks and one bonus trick:

  1. 能够在一行中有单独的分支.这是通过用空格而不是 !
    2. 分隔这些分支来实现的
  2. 能够使用 name=mux 制作别名，然后在名称末尾添加点来使用它，例如 mux. .你可以为你喜欢的元素起任何名字.
  3. 按ctrl+c停止录制后写入EOS.这是通过参数 -e
    4. 实现的
  1. ability to have separate branches in one line. This is achieved by just separating those branches with space instead of !
  2. ability to make alias with name=mux and later on using it with adding dot right at the end of name like mux. . You can make up any name for that element you like.
  3. Write EOS after hitting ctrl+c to stop the recording. This is achieved with parameter -e

最后输出到文件接收器，它只写你给文件的任何东西.

Finally the output goes to filesink which just writes anything you give it to file.

现在做作业:

  1. 根据需要使用元素 - v4l2、alsasrc

  1. Use your elements for what you need - v4l2, alsasrc

添加队列元素以添加缓冲和线程分离

Add queue elements to add buffering and thread separation

这篇关于Gstreamer 录制带音频的视频的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！

查看全文
相关文章
服务器开发最新文章
热门教程
热门工具
登录 关闭
扫码关注1秒登录
发送“验证码”获取 | 15天全站免登陆