Audacity  2.2.2
ImportGStreamer.cpp
Go to the documentation of this file.
1 /**********************************************************************
2 
3 Audacity: A Digital Audio Editor
4 
5 ImportGStreamer.cpp
6 
7 Copyright 2008 LRN
8 Based on ImportFFmpeg.cpp by LRN
9 
10 Rework for gstreamer 1.0 by LLL
11 
12 Licensed under the GNU General Public License v2 or later
13 
14 *//****************************************************************//****************************************************************//*******************************************************************/
25 
26 #include "../Audacity.h" // needed before GStreamer.h
27 #include <wx/window.h>
28 #include <wx/log.h>
29 
30 #if defined(USE_GSTREAMER)
31 
32 #include "../MemoryX.h"
33 
34 #define DESC _("GStreamer-compatible files")
35 
36 
37 // On Windows we don't have configure script to turn this on or off,
38 // so let's use msw-specific pragma to add required libraries.
39 // Of course, library search path still has to be updated manually
40 #if defined(__WXMSW__)
41 # pragma comment(lib,"gstreamer-1.0.lib")
42 # pragma comment(lib,"gstapp-1.0.lib")
43 # pragma comment(lib,"gstbase-1.0.lib")
44 # pragma comment(lib,"glib-2.0.lib")
45 # pragma comment(lib,"gobject-2.0.lib")
46 #endif
47 
48 // all the includes live here by default
49 #include "../AudacityException.h"
50 #include "../SampleFormat.h"
51 #include "../Tags.h"
52 #include "../Internat.h"
53 #include "../WaveTrack.h"
54 #include "Import.h"
55 #include "ImportPlugin.h"
56 #include "ImportGStreamer.h"
57 
58 extern "C"
59 {
60 // #include <gio/gio.h>
61  #include <gst/app/gstappsink.h>
62  #include <gst/audio/audio-format.h>
63 }
64 
65 // Convenience macros
66 #define AUDCTX "audacity::context"
67 #define GETCTX(o) (GStreamContext *) g_object_get_data(G_OBJECT((o)), AUDCTX)
68 #define SETCTX(o, c) g_object_set_data(G_OBJECT((o)), AUDCTX, (gpointer) (c))
69 #define WARN(e, msg) GST_ELEMENT_WARNING((e), STREAM, FAILED, msg, (NULL));
70 
71 // Capabilities that Audacity can handle
72 //
73 // This resolves to: (on little endien)
74 //
75 // "audio/x-raw, "
76 // "format = (string) {S16LE, S24_32LE, F32LE}, "
77 // "rate = (int) [ 1, max ]",
78 // "channels = (int) [ 1, max ]"
79 
80 static GstStaticCaps supportedCaps =
81  GST_STATIC_CAPS(
82  GST_AUDIO_CAPS_MAKE(
83  "{"
84  GST_AUDIO_NE(S16) ", "
85  GST_AUDIO_NE(S24_32) ", "
86  GST_AUDIO_NE(F32)
87  "}"
88  )
89  );
90 
91 struct g_mutex_locker
92 {
93  explicit g_mutex_locker(GMutex &mutex_)
94  : mutex(mutex_)
95  {
96  g_mutex_lock(&mutex);
97  }
98 
99  ~g_mutex_locker()
100  {
101  g_mutex_unlock(&mutex);
102  }
103 
104  GMutex &mutex;
105 };
106 
107 template<typename T, void(*Fn)(T*)> struct Deleter {
108  inline void operator() (void *p) const
109  {
110  if (p)
111  Fn(static_cast<T*>(p));
112  }
113 };
114 using GstString = std::unique_ptr < gchar, Deleter<void, g_free> > ;
115 using GErrorHandle = std::unique_ptr < GError, Deleter<GError, g_error_free> > ;
116 
117 using ParseFn = void (*)(GstMessage *message, GError **gerror, gchar **debug);
118 inline void GstMessageParse(ParseFn fn, GstMessage *msg, GErrorHandle &err, GstString &debug)
119 {
120  GError *error;
121  gchar *string;
122  fn(msg, &error, &string);
123  err.reset(error);
124  debug.reset(string);
125 }
126 
127 // Context used for private stream data
128 struct GStreamContext
129 {
130  GstElement *mConv{}; // Audio converter
131  GstElement *mSink{}; // Application sink
132  bool mUse{}; // True if this stream should be imported
133  TrackHolders mChannels; // Array of WaveTrack pointers, one for each channel
134  unsigned mNumChannels{}; // Number of channels
135  gdouble mSampleRate{}; // Sample rate
136  GstString mType; // Audio type
137  sampleFormat mFmt{ floatSample }; // Sample format
138  gint64 mPosition{}; // Start position of stream
139  gint64 mDuration{}; // Duration of stream
140  GstElement *mPipeline{};
141 
142  GStreamContext() {}
143  ~GStreamContext()
144  {
145  // Remove the appsink element
146  if (mSink)
147  {
148  gst_bin_remove(GST_BIN(mPipeline), mSink);
149  }
150 
151  // Remove the audioconvert element
152  if (mConv)
153  {
154  gst_bin_remove(GST_BIN(mPipeline), mConv);
155  }
156  }
157 };
158 
159 // For RAII on gst objects
160 template<typename T> using GstObjHandle =
161  std::unique_ptr < T, Deleter<void, gst_object_unref > > ;
162 
164 class GStreamerImportFileHandle final : public ImportFileHandle
165 {
166 public:
167  GStreamerImportFileHandle(const wxString & name);
168  virtual ~GStreamerImportFileHandle();
169 
172  bool Init();
173 
174  wxString GetFileDescription() override;
175  ByteCount GetFileUncompressedBytes() override;
176 
179  wxInt32 GetStreamCount() override;
180 
183  const wxArrayString &GetStreamInfo() override;
184 
188  void SetStreamUsage(wxInt32 index, bool use) override;
189 
192  int Import(TrackFactory *trackFactory,
193  TrackHolders &outTracks,
194  Tags *tags) override;
195 
196  // =========================================================================
197  // Handled within the gstreamer threads
198  // =========================================================================
199 
202  void OnPadAdded(GstPad *pad);
203 
206  void OnPadRemoved(GstPad *pad);
207 
211  bool ProcessBusMessage(bool & success);
212 
216  void OnTag(GstAppSink *appsink, GstTagList *tags);
217 
221  void OnNewSample(GStreamContext *c, GstSample *sample);
222 
223 private:
224  wxArrayString mStreamInfo;
225  Tags mTags;
226  TrackFactory *mTrackFactory;
227 
228  GstString mUri;
229  GstObjHandle<GstElement> mPipeline;
230  GstObjHandle<GstBus> mBus;
231  GstElement *mDec;
232  bool mAsyncDone;
233 
234  GMutex mStreamsLock;
235  std::vector<std::unique_ptr<GStreamContext>> mStreams;
236 };
237 
240 class GStreamerImportPlugin final : public ImportPlugin
241 {
242 public:
245 
247  virtual ~GStreamerImportPlugin();
248 
249  wxString GetPluginFormatDescription();
250 
251  wxString GetPluginStringID();
252 
253  wxArrayString GetSupportedExtensions();
254 
256  std::unique_ptr<ImportFileHandle> Open(const wxString &Filename) override;
257 };
258 
259 // ============================================================================
260 // Initialization
261 // ============================================================================
262 
263 // ----------------------------------------------------------------------------
264 // Instantiate GStreamerImportPlugin and add to the list of known importers
265 void
267  UnusableImportPluginList & WXUNUSED(unusableImportPluginList))
268 {
269  wxLogMessage(_TS("Audacity is built against GStreamer version %d.%d.%d-%d"),
270  GST_VERSION_MAJOR,
271  GST_VERSION_MINOR,
272  GST_VERSION_MICRO,
273  GST_VERSION_NANO);
274 
275  // Initialize gstreamer
276  GErrorHandle error;
277  bool initError;
278  {
279  int argc = 0;
280  char **argv = NULL;
281  GError *ee;
282  initError = !gst_init_check(&argc, &argv, &ee);
283  error.reset(ee);
284  }
285  if ( initError )
286  {
287  wxLogMessage(wxT("Failed to initialize GStreamer. Error %d: %s"),
288  error.get()->code,
289  wxString::FromUTF8(error.get()->message));
290  return;
291  }
292 
293  guint major, minor, micro, nano;
294  gst_version(&major, &minor, &micro, &nano);
295  wxLogMessage(wxT("Linked to GStreamer version %d.%d.%d-%d"),
296  major,
297  minor,
298  micro,
299  nano);
300 
301  // Instantiate plugin
302  auto plug = std::make_unique<GStreamerImportPlugin>();
303 
304  // No supported extensions...no gstreamer plugins installed
305  if (plug->GetSupportedExtensions().GetCount() == 0)
306  return;
307 
308  // Add to list of importers
309  importPluginList.push_back( std::move(plug) );
310 }
311 
312 // ============================================================================
313 // GStreamerImportPlugin Class
314 // ============================================================================
315 
316 // ----------------------------------------------------------------------------
317 // Constructor
318 GStreamerImportPlugin::GStreamerImportPlugin()
319 : ImportPlugin(wxArrayString())
320 {
321 }
322 
323 // ----------------------------------------------------------------------------
324 // Destructor
325 GStreamerImportPlugin::~GStreamerImportPlugin()
326 {
327 }
328 
329 // ----------------------------------------------------------------------------
330 // Return the plugin description
331 wxString
332 GStreamerImportPlugin::GetPluginFormatDescription()
333 {
334  return DESC;
335 }
336 
337 // ----------------------------------------------------------------------------
338 // Return the plugin name
339 wxString
340 GStreamerImportPlugin::GetPluginStringID()
341 {
342  return wxT("gstreamer");
343 }
344 
345 // Obtains a list of supported extensions from typefind factories
346 // TODO: improve the list. It is obviously incomplete.
347 wxArrayString
348 GStreamerImportPlugin::GetSupportedExtensions()
349 {
350  // We refresh the extensions each time this is called in case the
351  // user had installed additional gstreamer plugins while Audacity
352  // was active.
353  mExtensions.Empty();
354 
355  // Gather extensions from all factories that support audio
356  {
357  std::unique_ptr < GList, Deleter<GList, gst_plugin_feature_list_free> >
358  factories{ gst_type_find_factory_get_list() };
359 
360  for (GList *list = factories.get(); list != NULL; list = g_list_next(list))
361  {
362  GstTypeFindFactory *factory = GST_TYPE_FIND_FACTORY(list->data);
363 
364  // We need the capabilities to determine if it handles audio
365  GstCaps *caps = gst_type_find_factory_get_caps(factory);
366  if (!caps)
367  {
368  continue;
369  }
370 
371  // Check each structure in the caps for audio
372  for (guint c = 0, clen = gst_caps_get_size(caps); c < clen; c++)
373  {
374  // Bypass if it isn't for audio
375  GstStructure *str = gst_caps_get_structure(caps, c);
376  if (!g_str_has_prefix(gst_structure_get_name(str), "audio"))
377  {
378  continue;
379  }
380 
381  // This factory can handle audio, so get the extensions
382  const gchar *const *extensions = gst_type_find_factory_get_extensions(factory);
383  if (!extensions)
384  {
385  continue;
386  }
387 
388  // Add each extension to the list
389  for (guint i = 0; extensions[i] != NULL; i++)
390  {
391  wxString extension = wxString::FromUTF8(extensions[i]);
392  if (mExtensions.Index(extension, false) == wxNOT_FOUND)
393  {
394  mExtensions.Add(extension);
395  }
396  }
397  }
398  }
399  }
400 
401  // Get them in a decent order
402  mExtensions.Sort();
403 
404  // Log it for debugging
405  wxString extensions = wxT("Extensions:");
406  for (size_t i = 0; i < mExtensions.GetCount(); i++)
407  {
408  extensions = extensions + wxT(" ") + mExtensions[i];
409  }
410  wxLogMessage(wxT("%s"), extensions);
411 
412  return mExtensions;
413 }
414 
415 // ----------------------------------------------------------------------------
416 // Open the file and return an importer "file handle"
417 std::unique_ptr<ImportFileHandle> GStreamerImportPlugin::Open(const wxString &filename)
418 {
419  auto handle = std::make_unique<GStreamerImportFileHandle>(filename);
420 
421  // Initialize the handle
422  if (!handle->Init())
423  {
424  return nullptr;
425  }
426 
427  // This std::move is needed to "upcast" the pointer type
428  return std::move(handle);
429 }
430 
431 // ============================================================================
432 // GStreamerImportFileHandle Class
433 // ============================================================================
434 
435 // ----------------------------------------------------------------------------
436 // The following methods/functions run within gstreamer thread contexts. No
437 // interaction with wxWidgets should be allowed. See explanation at the top
438 // of this file.
439 // ----------------------------------------------------------------------------
440 
441 // ----------------------------------------------------------------------------
442 // Filter out any video streams
443 //
444 // LRN found that by doing this here, video streams aren't decoded thus
445 // reducing the time/processing needed to extract the audio streams.
446 //
447 // This "gint" is really GstAutoplugSelectResult enum
448 static gint
449 GStreamerAutoplugSelectCallback(GstElement * WXUNUSED(element),
450  GstPad * WXUNUSED(pad),
451  GstCaps * WXUNUSED(caps),
452  GstElementFactory *factory,
453  gpointer WXUNUSED(data))
454 {
455  // Check factory class
456  const gchar *fclass = gst_element_factory_get_klass(factory);
457 
458  // Skip video decoding
459  if (g_strrstr(fclass,"Video"))
460  {
461  return 2; // GST_AUTOPLUG_SELECT_SKIP
462  }
463 
464  return 0; // GST_AUTOPLUG_SELECT_TRY
465 }
466 
467 // ----------------------------------------------------------------------------
468 // Handle the "pad-added" signal from uridecodebin
469 static void
470 GStreamerPadAddedCallback(GstElement * WXUNUSED(element),
471  GstPad *pad,
472  gpointer data)
473 {
474  ((GStreamerImportFileHandle *) data)->OnPadAdded(pad);
475 
476  return;
477 }
478 
479 // ----------------------------------------------------------------------------
480 // Handle the "pad-removed" signal from uridecodebin
481 static void
482 GStreamerPadRemovedCallback(GstElement * WXUNUSED(element),
483  GstPad *pad,
484  gpointer data)
485 {
486  ((GStreamerImportFileHandle *) data)->OnPadRemoved(pad);
487 
488  return;
489 }
490 
491 // ----------------------------------------------------------------------------
492 // Handle the "NEW-sample" signal from uridecodebin
493 inline void GstSampleUnref(GstSample *p) { gst_sample_unref(p); } // I can't use the static function name directly...
494 
495 static GstFlowReturn
496 GStreamerNewSample(GstAppSink *appsink, gpointer data)
497 {
498  // Don't let C++ exceptions propagate through GStreamer
499  return GuardedCall< GstFlowReturn > ( [&] {
501  static GMutex mutex;
502 
503  // Get the sample
504  std::unique_ptr < GstSample, Deleter< GstSample, GstSampleUnref> >
505  sample{ gst_app_sink_pull_sample(appsink) };
506 
507  // We must single thread here to prevent concurrent use of the
508  // Audacity track functions.
509  g_mutex_locker locker{ mutex };
510 
511  handle->OnNewSample(GETCTX(appsink), sample.get());
512 
513  return GST_FLOW_OK;
514  }, MakeSimpleGuard(GST_FLOW_ERROR) );
515 }
516 
517 // ----------------------------------------------------------------------------
518 // AppSink callbacks are used instead of signals to reduce processing
519 static GstAppSinkCallbacks AppSinkCallbacks =
520 {
521  NULL, // eos
522  NULL, // new_preroll
523  GStreamerNewSample // new_sample
524 };
525 
526 static GstAppSinkCallbacks AppSinkBitBucket =
527 {
528  NULL, // eos
529  NULL, // new_preroll
530  NULL // new_sample
531 };
532 
533 inline void GstCapsUnref(GstCaps *p) { gst_caps_unref(p); } // I can't use the static function name directly...
534 using GstCapsHandle = std::unique_ptr < GstCaps, Deleter<GstCaps, GstCapsUnref> >;
535 
536 // ----------------------------------------------------------------------------
537 // Handle the "pad-added" message
538 void
539 GStreamerImportFileHandle::OnPadAdded(GstPad *pad)
540 {
541  GStreamContext *c{};
542 
543  {
544  // Retrieve the stream caps...skip stream if unavailable
545  GstCaps *caps = gst_pad_get_current_caps(pad);
546  GstCapsHandle handle{ caps };
547 
548  if (!caps)
549  {
550  WARN(mPipeline.get(), ("OnPadAdded: unable to retrieve stream caps"));
551  return;
552  }
553 
554  // Get the caps structure...no need to release
555  GstStructure *str = gst_caps_get_structure(caps, 0);
556  if (!str)
557  {
558  WARN(mPipeline.get(), ("OnPadAdded: unable to retrieve caps structure"));
559  return;
560  }
561 
562  // Only accept audio streams...no need to release
563  const gchar *name = gst_structure_get_name(str);
564  if (!g_strrstr(name, "audio"))
565  {
566  WARN(mPipeline.get(), ("OnPadAdded: bypassing '%s' stream", name));
567  return;
568  }
569 
570  {
571  // Allocate a NEW stream context
572  auto uc = std::make_unique<GStreamContext>();
573  c = uc.get();
574  if (!c)
575  {
576  WARN(mPipeline.get(), ("OnPadAdded: unable to allocate stream context"));
577  return;
578  }
579 
580  // Set initial state
581  c->mUse = true;
582 
583  // Always add it to the context list to keep the number of contexts
584  // in sync with the number of streams
585  g_mutex_locker{ mStreamsLock };
586  // Pass the buck from uc
587  mStreams.push_back(std::move(uc));
588  }
589 
590  c->mPipeline = mPipeline.get();
591 
592  // Need pointer to context during pad removal (pad-remove signal)
593  SETCTX(pad, c);
594 
595  // Save the stream's start time and duration
596  gst_pad_query_position(pad, GST_FORMAT_TIME, &c->mPosition);
597  gst_pad_query_duration(pad, GST_FORMAT_TIME, &c->mDuration);
598 
599  // Retrieve the number of channels and validate
600  gint channels = -1;
601  gst_structure_get_int(str, "channels", &channels);
602  if (channels <= 0)
603  {
604  WARN(mPipeline.get(), ("OnPadAdded: channel count is invalid %d", channels));
605  return;
606  }
607  c->mNumChannels = channels;
608 
609  // Retrieve the sample rate and validate
610  gint rate = -1;
611  gst_structure_get_int(str, "rate", &rate);
612  if (rate <= 0)
613  {
614  WARN(mPipeline.get(), ("OnPadAdded: sample rate is invalid %d", rate));
615  return;
616  }
617  c->mSampleRate = (double)rate;
618 
619  c->mType.reset(g_strdup(name));
620  if (!c->mType)
621  {
622  WARN(mPipeline.get(), ("OnPadAdded: unable to allocate audio type"));
623  return;
624  }
625 
626  // Done with capabilities
627  }
628 
629  // Create audioconvert element
630  c->mConv = gst_element_factory_make("audioconvert", NULL);
631  if (!c->mConv)
632  {
633  WARN(mPipeline.get(), ("OnPadAdded: failed to create audioconvert element"));
634  return;
635  }
636 
637  // Create appsink element
638  c->mSink = gst_element_factory_make("appsink", NULL);
639  if (!c->mSink)
640  {
641  WARN(mPipeline.get(), ("OnPadAdded: failed to create appsink element"));
642  return;
643  }
644  SETCTX(c->mSink, c);
645 
646  // Set the appsink callbacks and add the context pointer
647  gst_app_sink_set_callbacks(GST_APP_SINK(c->mSink), &AppSinkCallbacks, this, NULL);
648 
649  {
650  // Set the capabilities that we desire
651  GstCaps *caps = gst_static_caps_get(&supportedCaps);
652  GstCapsHandle handle{ caps };
653  if (!caps)
654  {
655  WARN(mPipeline.get(), ("OnPadAdded: failed to create static caps"));
656  return;
657  }
658  gst_app_sink_set_caps(GST_APP_SINK(c->mSink), caps);
659  }
660 
661  // Do not sync to the clock...process as quickly as possible
662  gst_base_sink_set_sync(GST_BASE_SINK(c->mSink), FALSE);
663 
664  // Don't drop buffers...allow queue to build unfettered
665  gst_app_sink_set_drop(GST_APP_SINK(c->mSink), FALSE);
666 
667  // Add both elements to the pipeline
668  gst_bin_add_many(GST_BIN(mPipeline.get()), c->mConv, c->mSink, NULL);
669 
670  // Link them together
671  if (!gst_element_link(c->mConv, c->mSink))
672  {
673  WARN(mPipeline.get(), ("OnPadAdded: failed to link autioconvert and appsink"));
674  return;
675  }
676 
677  // Link the audiconvert sink pad to the src pad
678  GstPadLinkReturn ret = GST_PAD_LINK_OK;
679  {
680  GstObjHandle<GstPad> convsink{ gst_element_get_static_pad(c->mConv, "sink") };
681  if (convsink)
682  ret = gst_pad_link(pad, convsink.get());
683  if (!convsink || ret != GST_PAD_LINK_OK)
684  {
685  WARN(mPipeline.get(), ("OnPadAdded: failed to link uridecodebin to audioconvert - %d", ret));
686  return;
687  }
688  }
689 
690  // Synchronize audioconvert state with parent
691  if (!gst_element_sync_state_with_parent(c->mConv))
692  {
693  WARN(mPipeline.get(), ("OnPadAdded: unable to sync audioconvert state"));
694  return;
695  }
696 
697  // Synchronize appsink state with parent
698  if (!gst_element_sync_state_with_parent(c->mSink))
699  {
700  WARN(mPipeline.get(), ("OnPadAdded: unable to sync appaink state"));
701  return;
702  }
703 
704  return;
705 }
706 
707 // ----------------------------------------------------------------------------
708 // Process the "pad-removed" signal from uridecodebin
709 void
710 GStreamerImportFileHandle::OnPadRemoved(GstPad *pad)
711 {
712  GStreamContext *c = GETCTX(pad);
713 
714  // Set audioconvert and appsink states to NULL
715  gst_element_set_state(c->mSink, GST_STATE_NULL);
716  gst_element_set_state(c->mConv, GST_STATE_NULL);
717 
718  // Unlink audioconvert -> appsink
719  gst_element_unlink(c->mConv, c->mSink);
720 
721  // Remove the pads from the pipeilne
722  gst_bin_remove_many(GST_BIN(mPipeline.get()), c->mConv, c->mSink, NULL);
723 
724  // And reset context
725  c->mConv = NULL;
726  c->mSink = NULL;
727 
728  return;
729 }
730 
731 // ----------------------------------------------------------------------------
732 // Handle the "NEW-sample" message
733 void
734 GStreamerImportFileHandle::OnNewSample(GStreamContext *c, GstSample *sample)
735 {
736  // Allocate NEW tracks
737  //
738  // It is done here because, at least in the case of chained oggs,
739  // not all streams are known ahead of time.
740  if (c->mChannels.empty())
741  {
742  // Get the sample format...no need to release caps or structure
743  GstCaps *caps = gst_sample_get_caps(sample);
744  GstStructure *str = gst_caps_get_structure(caps, 0);
745  const gchar *fmt = gst_structure_get_string(str, "format");
746  if (!fmt)
747  {
748  WARN(mPipeline.get(), ("OnNewSample: missing audio format"));
749  return;
750  }
751 
752  // Determinate sample format based on negotiated format
753  if (strcmp(fmt, GST_AUDIO_NE(S16)) == 0)
754  {
755  c->mFmt = int16Sample;
756  }
757  else if (strcmp(fmt, GST_AUDIO_NE(S24_32)) == 0)
758  {
759  c->mFmt = int24Sample;
760  }
761  else if (strcmp(fmt, GST_AUDIO_NE(F32)) == 0)
762  {
763  c->mFmt = floatSample;
764  }
765  else
766  {
767  // This shouldn't really happen since audioconvert will only give us
768  // the formats we said we could handle.
769  WARN(mPipeline.get(), ("OnNewSample: unrecognized sample format %s", fmt));
770  return;
771  }
772 
773  // Allocate the track array
774  c->mChannels.resize(c->mNumChannels);
775  if (c->mChannels.size() != c->mNumChannels)
776  {
777  WARN(mPipeline.get(), ("OnNewSample: unable to allocate track array"));
778  return;
779  }
780 
781  // Allocate all channels
782  for (int ch = 0; ch < c->mNumChannels; ch++)
783  {
784  // Create a track
785  c->mChannels[ch] = mTrackFactory->NewWaveTrack(c->mFmt, c->mSampleRate);
786  if (!c->mChannels[ch])
787  {
788  WARN(mPipeline.get(), ("OnNewSample: unable to create track"));
789  return;
790  }
791  }
792 
793  // Set to stereo if there's exactly 2 channels
794  if (c->mNumChannels == 2)
795  {
796  c->mChannels[0]->SetChannel(Track::LeftChannel);
797  c->mChannels[1]->SetChannel(Track::RightChannel);
798  c->mChannels[0]->SetLinked(true);
799  }
800  }
801 
802  // Get the buffer for the sample...no need to release
803  GstBuffer *buffer = gst_sample_get_buffer(sample);
804  if (!buffer)
805  {
806  // No buffer...not sure if this is an error or not,
807  // but we can't do anything else, so just bail silently.
808  return;
809  }
810 
811  // Map the buffer
812  GstMapInfo info;
813  if (!gst_buffer_map(buffer, &info, GST_MAP_READ))
814  {
815  WARN(mPipeline.get(), ("OnNewSample: mapping buffer failed"));
816  return;
817  }
818  auto cleanup = finally([&]{
819  // Release buffer
820  gst_buffer_unmap(buffer, &info);
821  });
822 
823  // Cache a few items
824  auto nChannels = c->mNumChannels;
825  sampleFormat fmt = c->mFmt;
826  samplePtr data = (samplePtr) info.data;
827  size_t samples = info.size / nChannels / SAMPLE_SIZE(fmt);
828 
829  // Add sample data to tracks...depends on interleaved src data
830  for (int chn = 0; chn < nChannels; chn++)
831  {
832  // Append one channel
833  c->mChannels[chn]->Append(data,
834  fmt,
835  samples,
836  nChannels);
837 
838  // Bump src to next channel
839  data += SAMPLE_SIZE(fmt);
840  }
841 
842  return;
843 }
844 
845 // ----------------------------------------------------------------------------
846 // The following methods run within the main thread.
847 // ----------------------------------------------------------------------------
848 
849 // ----------------------------------------------------------------------------
850 // Constructor
851 GStreamerImportFileHandle::GStreamerImportFileHandle(const wxString & name)
852 : ImportFileHandle(name)
853 {
854  mDec = NULL;
855  mTrackFactory = NULL;
856  mAsyncDone = false;
857 
858  g_mutex_init(&mStreamsLock);
859 }
860 
861 // ----------------------------------------------------------------------------
862 // Destructor
863 GStreamerImportFileHandle::~GStreamerImportFileHandle()
864 {
865  // Make sure the pipeline isn't running
866  if (mPipeline)
867  {
868  gst_element_set_state(mPipeline.get(), GST_STATE_NULL);
869  }
870 
871  // Delete all of the contexts
872  if (mStreams.size())
873  {
874  // PRL: is the FIFO destruction order important?
875  // If not, then you could simply clear mStreams.
876 
877  {
878  g_mutex_locker locker{ mStreamsLock };
879  while (mStreams.size() > 0)
880  {
881  // remove context from the array
882  mStreams.erase(mStreams.begin());
883  }
884  }
885 
886  // Done with the context array
887  }
888 
889  // Release the decoder
890  if (mDec != NULL)
891  {
892  gst_bin_remove(GST_BIN(mPipeline.get()), mDec);
893  }
894 
895  g_mutex_clear(&mStreamsLock);
896 }
897 
898 // ----------------------------------------------------------------------------
899 // Return number of readable audio streams in the file
900 wxInt32
901 GStreamerImportFileHandle::GetStreamCount()
902 {
903  return mStreamInfo.GetCount();
904 }
905 
906 // ----------------------------------------------------------------------------
907 // Return array of strings - descriptions of the streams
908 const wxArrayString &
909 GStreamerImportFileHandle::GetStreamInfo()
910 {
911  return mStreamInfo;
912 }
913 
914 // ----------------------------------------------------------------------------
915 // Mark streams to process as selected by the user
916 void
917 GStreamerImportFileHandle::SetStreamUsage(wxInt32 index, bool use)
918 {
919  g_mutex_locker locker{ mStreamsLock };
920  if ((guint) index < mStreams.size())
921  {
922  GStreamContext *c = mStreams[index].get();
923  c->mUse = use;
924  }
925 }
926 
927 // ----------------------------------------------------------------------------
928 // Initialize importer
929 bool
930 GStreamerImportFileHandle::Init()
931 {
932  // Create a URI from the filename
933  mUri.reset(g_strdup_printf("file:///%s", mFilename.ToUTF8().data()));
934  if (!mUri)
935  {
936  wxLogMessage(wxT("GStreamerImport couldn't create URI"));
937  return false;
938  }
939 
940  // Create a pipeline
941  mPipeline.reset(gst_pipeline_new("pipeline"));
942 
943  // Get its bus
944  mBus.reset(gst_pipeline_get_bus(GST_PIPELINE(mPipeline.get())));
945 
946  // Create uridecodebin and set up signal handlers
947  mDec = gst_element_factory_make("uridecodebin", "decoder");
948  g_signal_connect(mDec, "autoplug-select", G_CALLBACK(GStreamerAutoplugSelectCallback), (gpointer) this);
949  g_signal_connect(mDec, "pad-added", G_CALLBACK(GStreamerPadAddedCallback), (gpointer) this);
950  g_signal_connect(mDec, "pad-removed", G_CALLBACK(GStreamerPadRemovedCallback), (gpointer) this);
951 
952  // Set the URI
953  g_object_set(G_OBJECT(mDec), "uri", mUri, NULL);
954 
955  // Add the decoder to the pipeline
956  if (!gst_bin_add(GST_BIN(mPipeline.get()), mDec))
957  {
958  AudacityMessageBox(_("Unable to add decoder to pipeline"),
959  _("GStreamer Importer"));
960 
961  // Cleanup expected to occur in destructor
962  return false;
963  }
964 
965  // Run the pipeline
966  GstStateChangeReturn state = gst_element_set_state(mPipeline.get(), GST_STATE_PAUSED);
967  if (state == GST_STATE_CHANGE_FAILURE)
968  {
969  AudacityMessageBox(_("Unable to set stream state to paused."),
970  _("GStreamer Importer"));
971  return false;
972  }
973 
974  // Collect info while the stream is prerolled
975  //
976  // Unfortunately, for some files this may cause a slight "pause" in the GUI
977  // without a progress dialog appearing. Not much can be done about it other
978  // than throwing up an additional progress dialog and displaying two dialogs
979  // may be confusing to the users.
980 
981  // Process messages until we get an error or the ASYNC_DONE message is received
982  bool success;
983  while (ProcessBusMessage(success) && success)
984  {
985  // Give wxWidgets a chance to do housekeeping
986  wxSafeYield();
987  }
988 
989  // Build the stream info array
990  g_mutex_locker locker{ mStreamsLock };
991  for (guint i = 0; i < mStreams.size(); i++)
992  {
993  GStreamContext *c = mStreams[i].get();
994 
995  // Create stream info string
996  wxString strinfo;
997  strinfo.Printf(wxT("Index[%02d], Type[%s], Channels[%d], Rate[%d]"),
998  (unsigned int) i,
999  wxString::FromUTF8(c->mType.get()),
1000  (int) c->mNumChannels,
1001  (int) c->mSampleRate);
1002  mStreamInfo.Add(strinfo);
1003  }
1004 
1005  return success;
1006 }
1007 
1008 // ----------------------------------------------------------------------------
1009 // Return file dialog filter description
1010 wxString
1011 GStreamerImportFileHandle::GetFileDescription()
1012 {
1013  return DESC;
1014 }
1015 
1016 // ----------------------------------------------------------------------------
1017 // Return number of uncompressed bytes in file...doubtful this is possible
1018 auto
1019 GStreamerImportFileHandle::GetFileUncompressedBytes() -> ByteCount
1020 {
1021  return 0;
1022 }
1023 
1024 // ----------------------------------------------------------------------------
1025 // Import streams
1026 int
1027 GStreamerImportFileHandle::Import(TrackFactory *trackFactory,
1028  TrackHolders &outTracks,
1029  Tags *tags)
1030 {
1031  outTracks.clear();
1032 
1033  // Save track factory pointer
1034  mTrackFactory = trackFactory;
1035 
1036  // Create the progrress dialog
1037  CreateProgress();
1038 
1039  // Block streams that are to be bypassed
1040  bool haveStreams = false;
1041  {
1042  g_mutex_locker locker{ mStreamsLock };
1043  for (guint i = 0; i < mStreams.size(); i++)
1044  {
1045  GStreamContext *c = mStreams[i].get();
1046 
1047  // Did the user choose to skip this stream?
1048  if (!c->mUse)
1049  {
1050  // Get the audioconvert sink pad and unlink
1051  {
1052  GstObjHandle<GstPad> convsink{ gst_element_get_static_pad(c->mConv, "sink") };
1053 
1054  {
1055  GstObjHandle<GstPad> convpeer{ gst_pad_get_peer(convsink.get()) };
1056  gst_pad_unlink(convpeer.get(), convsink.get());
1057  }
1058 
1059  // Set bitbucket callbacks so the prerolled sample won't get processed
1060  // when we change the state to PLAYING
1061  gst_app_sink_set_callbacks(GST_APP_SINK(c->mSink), &AppSinkBitBucket, this, NULL);
1062 
1063  // Set state to playing for conv and sink so EOS gets processed
1064  gst_element_set_state(c->mConv, GST_STATE_PLAYING);
1065  gst_element_set_state(c->mSink, GST_STATE_PLAYING);
1066 
1067  // Send an EOS event to the pad to force them to drain
1068  gst_pad_send_event(convsink.get(), gst_event_new_eos());
1069 
1070  // Resync state with pipeline
1071  gst_element_sync_state_with_parent(c->mConv);
1072  gst_element_sync_state_with_parent(c->mSink);
1073 
1074  // Done with the pad
1075  }
1076 
1077  // Unlink audioconvert and appsink
1078  gst_element_unlink(c->mConv, c->mSink);
1079 
1080  // Remove them from the bin
1081  gst_bin_remove_many(GST_BIN(mPipeline.get()), c->mConv, c->mSink, NULL);
1082 
1083  // All done with them
1084  c->mConv = NULL;
1085  c->mSink = NULL;
1086 
1087  continue;
1088  }
1089 
1090  // We have a stream to process
1091  haveStreams = true;
1092  }
1093  }
1094 
1095  // Can't do much if we don't have any streams to process
1096  if (!haveStreams)
1097  {
1098  AudacityMessageBox(_("File doesn't contain any audio streams."),
1099  _("GStreamer Importer"));
1100  return ProgressResult::Failed;
1101  }
1102 
1103  // Get the ball rolling...
1104  GstStateChangeReturn state = gst_element_set_state(mPipeline.get(), GST_STATE_PLAYING);
1105  if (state == GST_STATE_CHANGE_FAILURE)
1106  {
1107  AudacityMessageBox(_("Unable to import file, state change failed."),
1108  _("GStreamer Importer"));
1109  return ProgressResult::Failed;
1110  }
1111 
1112  // Get the duration of the stream
1113  gint64 duration;
1114  gst_element_query_duration(mPipeline.get(), GST_FORMAT_TIME, &duration);
1115 
1116  // Handle bus messages and update progress while files is importing
1117  bool success = true;
1118  int updateResult = ProgressResult::Success;
1119  while (ProcessBusMessage(success) && success && updateResult == ProgressResult::Success)
1120  {
1121  gint64 position;
1122 
1123  // Update progress indicator and give user chance to abort
1124  if (gst_element_query_position(mPipeline.get(), GST_FORMAT_TIME, &position))
1125  {
1126  updateResult = mProgress->Update((wxLongLong_t) position,
1127  (wxLongLong_t) duration);
1128  }
1129  }
1130 
1131  // Disable pipeline
1132  gst_element_set_state(mPipeline.get(), GST_STATE_NULL);
1133 
1134  // Something bad happened
1135  if (!success || updateResult == ProgressResult::Failed || updateResult == ProgressResult::Cancelled)
1136  {
1137  return updateResult;
1138  }
1139 
1140  // Grab the streams lock
1141  g_mutex_locker locker{ mStreamsLock };
1142 
1143  // Count the total number of tracks collected
1144  unsigned outNumTracks = 0;
1145  for (guint s = 0; s < mStreams.size(); s++)
1146  {
1147  GStreamContext *c = mStreams[s].get();
1148  if (c)
1149  outNumTracks += c->mNumChannels;
1150  }
1151 
1152  // Create NEW tracks
1153  outTracks.resize(outNumTracks);
1154 
1155  // Copy audio from mChannels to newly created tracks (destroying mChannels in process)
1156  int trackindex = 0;
1157  for (guint s = 0; s < mStreams.size(); s++)
1158  {
1159  GStreamContext *c = mStreams[s].get();
1160  if (c->mNumChannels)
1161  {
1162  for (int ch = 0; ch < c->mNumChannels; ch++)
1163  {
1164  c->mChannels[ch]->Flush();
1165  outTracks[trackindex++] = std::move(c->mChannels[ch]);
1166  }
1167 
1168  c->mChannels.clear();
1169  }
1170  }
1171 
1172  // Set any tags found in the stream
1173  *tags = mTags;
1174 
1175  return updateResult;
1176 }
1177 
1178 // ----------------------------------------------------------------------------
1179 // Message handlers
1180 // ----------------------------------------------------------------------------
1181 
1182 inline void GstMessageUnref(GstMessage *p) { gst_message_unref(p); }
1183 
1184 // ----------------------------------------------------------------------------
1185 // Retrieve and process a bus message
1186 bool
1187 GStreamerImportFileHandle::ProcessBusMessage(bool & success)
1188 {
1189  bool cont = true;
1190 
1191  // Default to no errors
1192  success = true;
1193 
1194  // Get the next message
1195  std::unique_ptr < GstMessage, Deleter < GstMessage, GstMessageUnref > >
1196  msg{ gst_bus_timed_pop(mBus.get(), 100 * GST_MSECOND) };
1197 
1198  if (!msg)
1199  {
1200  // Timed out...not an error
1201  return cont;
1202  }
1203 
1204 #if defined(__WXDEBUG__)
1205  {
1206  GstString objname;
1207  if (msg->src != NULL)
1208  {
1209  objname.reset(gst_object_get_name(msg->src));
1210  }
1211 
1212  wxLogMessage(wxT("GStreamer: Got %s%s%s"),
1213  wxString::FromUTF8(GST_MESSAGE_TYPE_NAME(msg.get())),
1214  objname ? wxT(" from ") : wxT(""),
1215  objname ? wxString::FromUTF8(objname.get()) : wxT(""));
1216  }
1217 #endif
1218 
1219  // Handle based on message type
1220  switch (GST_MESSAGE_TYPE(msg.get()))
1221  {
1222  // Handle error message from gstreamer
1223  case GST_MESSAGE_ERROR:
1224  {
1225  GErrorHandle err;
1226  GstString debug;
1227 
1228  GstMessageParse(gst_message_parse_error, msg.get(), err, debug);
1229  if (err)
1230  {
1231  wxString m;
1232 
1233  m.Printf(wxT("%s%s%s"),
1234  wxString::FromUTF8(err.get()->message),
1235  debug ? wxT("\n") : wxT(""),
1236  debug ? wxString::FromUTF8(debug.get()) : wxT(""));
1237 #if defined(_DEBUG)
1238  AudacityMessageBox(wxString::Format(_("GStreamer Error: %s"), m));
1239 #else
1240  wxLogMessage(wxT("GStreamer Error: %s"), m);
1241 #endif
1242  }
1243 
1244  success = false;
1245  cont = false;
1246  }
1247  break;
1248 
1249  // Handle warning message from gstreamer
1250  case GST_MESSAGE_WARNING:
1251  {
1252  GErrorHandle err;
1253  GstString debug;
1254  GstMessageParse(gst_message_parse_warning, msg.get(), err, debug);
1255 
1256  if (err)
1257  {
1258  wxLogMessage(wxT("GStreamer Warning: %s%s%s"),
1259  wxString::FromUTF8(err.get()->message),
1260  debug ? wxT("\n") : wxT(""),
1261  debug ? wxString::FromUTF8(debug.get()) : wxT(""));
1262  }
1263  }
1264  break;
1265 
1266  // Handle warning message from gstreamer
1267  case GST_MESSAGE_INFO:
1268  {
1269  GErrorHandle err;
1270  GstString debug;
1271 
1272  GstMessageParse(gst_message_parse_info, msg.get(), err, debug);
1273  if (err)
1274  {
1275  wxLogMessage(wxT("GStreamer Info: %s%s%s"),
1276  wxString::FromUTF8(err.get()->message),
1277  debug ? wxT("\n") : wxT(""),
1278  debug ? wxString::FromUTF8(debug.get()) : wxT(""));
1279  }
1280  }
1281  break;
1282 
1283  // Handle metadata tags
1284  case GST_MESSAGE_TAG:
1285  {
1286  GstTagList *tags = NULL;
1287  auto cleanup = finally([&]{
1288  // Done with list
1289  if(tags) gst_tag_list_unref(tags);
1290  });
1291 
1292  // Retrieve tag list from message...just ignore failure
1293  gst_message_parse_tag(msg.get(), &tags);
1294  if (tags)
1295  {
1296  // Go process the list
1297  OnTag(GST_APP_SINK(GST_MESSAGE_SRC(msg.get())), tags);
1298  }
1299  }
1300  break;
1301 
1302  // Pre-roll is done...will happen for each group
1303  // (like with chained OGG files)
1304  case GST_MESSAGE_ASYNC_DONE:
1305  {
1306  // If this is the first async-done message, then tell
1307  // caller to end loop, but leave it active so that
1308  // gstreamer threads can still queue up.
1309  //
1310  // We'll receive multiple async-done messages for chained
1311  // ogg files, so ignore the message the 2nd and subsequent
1312  // occurrences.
1313  if (!mAsyncDone)
1314  {
1315  cont = false;
1316  mAsyncDone = true;
1317  }
1318 
1319  }
1320  break;
1321 
1322  // End of the stream (and all sub-streams)
1323  case GST_MESSAGE_EOS:
1324  {
1325  // Terminate loop
1326  cont = false;
1327  }
1328  break;
1329  }
1330 
1331  return cont;
1332 }
1333 
1334 // ----------------------------------------------------------------------------
1335 // Handle the "tag" message
1336 void
1337 GStreamerImportFileHandle::OnTag(GstAppSink * WXUNUSED(appsink), GstTagList *tags)
1338 {
1339  // Collect all of the associates tags
1340  for (guint i = 0, icnt = gst_tag_list_n_tags(tags); i < icnt; i++)
1341  {
1342  wxString string;
1343 
1344  // Get tag name...should always succeed...no need to release
1345  const gchar *name = gst_tag_list_nth_tag_name(tags, i);
1346  if (!name)
1347  {
1348  continue;
1349  }
1350 
1351  // For each tag, determine its type and retrieve if possible
1352  for (guint j = 0, jcnt = gst_tag_list_get_tag_size(tags, name); j < jcnt; j++)
1353  {
1354  const GValue *val;
1355 
1356  val = gst_tag_list_get_value_index(tags, name, j);
1357 
1358  if (G_VALUE_HOLDS_STRING(val))
1359  {
1360  string = wxString::FromUTF8(g_value_get_string(val));
1361  }
1362  else if (G_VALUE_HOLDS_UINT(val))
1363  {
1364  string.Printf(wxT("%u"), (unsigned int) g_value_get_uint(val));
1365  }
1366  else if (G_VALUE_HOLDS_DOUBLE(val))
1367  {
1368  string.Printf(wxT("%g"), g_value_get_double(val));
1369  }
1370  else if (G_VALUE_HOLDS_BOOLEAN(val))
1371  {
1372  string = g_value_get_boolean(val) ? wxT("true") : wxT("false");
1373  }
1374  else if (GST_VALUE_HOLDS_DATE_TIME(val))
1375  {
1376  GstDateTime *dt = (GstDateTime *) g_value_get_boxed(val);
1377  GstString str{ gst_date_time_to_iso8601_string(dt) };
1378  string = wxString::FromUTF8(str.get());
1379  }
1380  else if (G_VALUE_HOLDS(val, G_TYPE_DATE))
1381  {
1382  GstString str{ gst_value_serialize(val) };
1383  string = wxString::FromUTF8(str.get());
1384  }
1385  else
1386  {
1387  wxLogMessage(wxT("Tag %s has unhandled type: %s"),
1388  wxString::FromUTF8(name),
1389  wxString::FromUTF8(G_VALUE_TYPE_NAME(val)));
1390  continue;
1391  }
1392 
1393  // Translate known tag names
1394  wxString tag;
1395  if (strcmp(name, GST_TAG_TITLE) == 0)
1396  {
1397  tag = TAG_TITLE;
1398  }
1399  else if (strcmp(name, GST_TAG_ARTIST) == 0)
1400  {
1401  tag = TAG_ARTIST;
1402  }
1403  else if (strcmp(name, GST_TAG_ALBUM) == 0)
1404  {
1405  tag = TAG_ALBUM;
1406  }
1407  else if (strcmp(name, GST_TAG_TRACK_NUMBER) == 0)
1408  {
1409  tag = TAG_TRACK;
1410  }
1411  else if (strcmp(name, GST_TAG_DATE) == 0)
1412  {
1413  tag = TAG_YEAR;
1414  }
1415  else if (strcmp(name, GST_TAG_GENRE) == 0)
1416  {
1417  tag = TAG_GENRE;
1418  }
1419  else if (strcmp(name, GST_TAG_COMMENT) == 0)
1420  {
1421  tag = TAG_COMMENTS;
1422  }
1423  else
1424  {
1425  tag = wxString::FromUTF8(name);
1426  }
1427 
1428  if (jcnt > 1)
1429  {
1430  tag.Printf(wxT("%s:%d"), tag, j);
1431  }
1432 
1433  // Store the tag
1434  mTags.SetTag(tag, string);
1435  }
1436  }
1437 }
1438 #endif //USE_GSTREAMER
#define TAG_TRACK
Definition: Tags.h:63
An ImportFileHandle for data.
Definition: ImportPlugin.h:119
#define SAMPLE_SIZE(SampleFormat)
Definition: Types.h:198
#define _TS(s)
Definition: Internat.h:30
#define TAG_TITLE
Definition: Tags.h:60
int AudacityMessageBox(const wxString &message, const wxString &caption=AudacityMessageBoxCaptionStr(), long style=wxOK|wxCENTRE, wxWindow *parent=NULL, int x=wxDefaultCoord, int y=wxDefaultCoord)
Definition: ErrorDialog.h:92
#define TAG_ARTIST
Definition: Tags.h:61
void GetGStreamerImportPlugin(ImportPluginList &importPluginList, UnusableImportPluginList &unusableImportPluginList)
SimpleGuard< R > MakeSimpleGuard(R value)
Used to create a WaveTrack, or a LabelTrack.. Implementation of the functions of this class are dispe...
Definition: Track.h:862
#define DESC
sampleFormat
Definition: Types.h:188
char * samplePtr
Definition: Types.h:203
The interface that all file import "plugins" (if you want to call them that) must implement...
An ImportPlugin for GStreamer data.
ID3 Tags (for MP3)
Definition: Tags.h:70
Base class for FlacImportPlugin, LOFImportPlugin, MP3ImportPlugin, OggImportPlugin and PCMImportPlugi...
Definition: ImportPlugin.h:73
_("Move Track &Down")+wxT("\t")+(GetActiveProject() -> GetCommandManager() ->GetKeyFromName(wxT("TrackMoveDown")).Raw()), OnMoveTrack) POPUP_MENU_ITEM(OnMoveTopID, _("Move Track to &Top")+wxT("\t")+(GetActiveProject() ->GetCommandManager() ->GetKeyFromName(wxT("TrackMoveTop")).Raw()), OnMoveTrack) POPUP_MENU_ITEM(OnMoveBottomID, _("Move Track to &Bottom")+wxT("\t")+(GetActiveProject() ->GetCommandManager() ->GetKeyFromName(wxT("TrackMoveBottom")).Raw()), OnMoveTrack)#define SET_TRACK_NAME_PLUGIN_SYMBOLclass SetTrackNameCommand:public AudacityCommand
An ImportFileHandle for GStreamer data.
const wxChar * name
Definition: Distortion.cpp:94
#define TAG_COMMENTS
Definition: Tags.h:66
#define TAG_GENRE
Definition: Tags.h:65
An UnusableImportPlugin list.
An ImportPlugin list.
std::vector< std::unique_ptr< WaveTrack >> TrackHolders
Definition: ImportRaw.h:24
#define TAG_ALBUM
Definition: Tags.h:62
#define TAG_YEAR
Definition: Tags.h:64