Audacity 3.2.0
SBSMSEffect.cpp
Go to the documentation of this file.
1/**********************************************************************
2
3Audacity: A Digital Audio Editor
4
5SBSMSEffect.cpp
6
7Clayton Otey
8
9This class contains all of the common code for an
10effect that uses SBSMS to do its processing (TimeScale)
11
12**********************************************************************/
13#if USE_SBSMS
14#include "SBSMSEffect.h"
15#include "EffectOutputTracks.h"
16
17#include <math.h>
18
19#include "../LabelTrack.h"
20#include "SyncLock.h"
21#include "WaveClip.h"
22#include "WaveTrack.h"
23#include "TimeWarper.h"
24
25#include <cassert>
26
27enum {
29};
30
32{
33public:
35 {
36 processed = 0;
37 }
38
40 {
41 }
42
43 bool bPitch;
45 double ratio;
47 size_t blockSize;
55 std::unique_ptr<SBSMS> sbsms;
56 std::unique_ptr<SBSMSInterface> iface;
58
59 // Not required by callbacks, but makes for easier cleanup
60 std::unique_ptr<Resampler> resampler;
61 std::unique_ptr<SBSMSQuality> quality;
65
66 std::exception_ptr mpException {};
67};
68
69class SBSMSEffectInterface final : public SBSMSInterfaceSliding {
70public:
72 Slide *rateSlide, Slide *pitchSlide,
73 bool bReferenceInput,
74 const SampleCountType samples, long preSamples,
75 SBSMSQuality *quality)
76 : SBSMSInterfaceSliding(rateSlide,pitchSlide,bReferenceInput,samples,preSamples,quality)
77 {
78 this->resampler = resampler;
79 }
81
82 long samples(audio *buf, long n) {
83 return resampler->read(buf, n);
84 }
85
86protected:
87 Resampler *resampler;
88};
89
90long resampleCB(void *cb_data, SBSMSFrame *data)
91{
92 ResampleBuf *r = (ResampleBuf*) cb_data;
93
94 auto blockSize = limitSampleBufferSize(
96 r->end - r->offset
97 );
98
99 // Get the samples from the tracks and put them in the buffers.
100 // I don't know if we can safely propagate errors through sbsms, and it
101 // does not seem to let us report error codes, so use this roundabout to
102 // stop the effect early.
103 try {
105 (r->leftBuffer.get()), r->offset, blockSize);
107 (r->rightBuffer.get()), r->offset, blockSize);
108 }
109 catch ( ... ) {
110 // Save the exception object for re-throw when out of the library
111 r->mpException = std::current_exception();
112 data->size = 0;
113 return 0;
114 }
115
116 // convert to sbsms audio format
117 for(decltype(blockSize) i=0; i<blockSize; i++) {
118 r->buf[i][0] = r->leftBuffer[i];
119 r->buf[i][1] = r->rightBuffer[i];
120 }
121
122 data->buf = r->buf.get();
123 data->size = blockSize;
124 if(r->bPitch) {
125 float t0 = r->processed.as_float() / r->iface->getSamplesToInput();
126 float t1 = (r->processed + blockSize).as_float() / r->iface->getSamplesToInput();
127 data->ratio0 = r->iface->getStretch(t0);
128 data->ratio1 = r->iface->getStretch(t1);
129 } else {
130 data->ratio0 = r->ratio;
131 data->ratio1 = r->ratio;
132 }
133 r->processed += blockSize;
134 r->offset += blockSize;
135 return blockSize;
136}
137
138long postResampleCB(void *cb_data, SBSMSFrame *data)
139{
140 ResampleBuf *r = (ResampleBuf*) cb_data;
141 auto count = r->sbsms->read(r->iface.get(), r->SBSMSBuf.get(), r->SBSMSBlockSize);
142 data->buf = r->SBSMSBuf.get();
143 data->size = count;
144 data->ratio0 = 1.0 / r->ratio;
145 data->ratio1 = 1.0 / r->ratio;
146 return count;
147}
148
149void EffectSBSMS :: setParameters(double rateStartIn, double rateEndIn, double pitchStartIn, double pitchEndIn,
150 SlideType rateSlideTypeIn, SlideType pitchSlideTypeIn,
151 bool bLinkRatePitchIn, bool bRateReferenceInputIn, bool bPitchReferenceInputIn)
152{
153 this->rateStart = rateStartIn;
154 this->rateEnd = rateEndIn;
155 this->pitchStart = pitchStartIn;
156 this->pitchEnd = pitchEndIn;
157 this->bLinkRatePitch = bLinkRatePitchIn;
158 this->rateSlideType = rateSlideTypeIn;
159 this->pitchSlideType = pitchSlideTypeIn;
160 this->bRateReferenceInput = bRateReferenceInputIn;
161 this->bPitchReferenceInput = bPitchReferenceInputIn;
162}
163
164void EffectSBSMS::setParameters(double tempoRatio, double pitchRatio)
165{
166 setParameters(tempoRatio, tempoRatio, pitchRatio, pitchRatio,
167 SlideConstant, SlideConstant, false, false, false);
168}
169
170std::unique_ptr<TimeWarper> createTimeWarper(double t0, double t1, double duration,
171 double rateStart, double rateEnd, SlideType rateSlideType)
172{
173 if (rateStart == rateEnd || rateSlideType == SlideConstant)
174 return std::make_unique<LinearTimeWarper>(
175 t0, t0, t1, t0 + duration);
176 else if(rateSlideType == SlideLinearInputRate)
177 return std::make_unique<LinearInputRateTimeWarper>(
178 t0, t1, rateStart, rateEnd);
179 else if(rateSlideType == SlideLinearOutputRate)
180 return std::make_unique<LinearOutputRateTimeWarper>(
181 t0, t1, rateStart, rateEnd);
182 else if(rateSlideType == SlideLinearInputStretch)
183 return std::make_unique<LinearInputStretchTimeWarper>(
184 t0, t1, rateStart, rateEnd);
185 else if(rateSlideType == SlideLinearOutputStretch)
186 return std::make_unique<LinearOutputStretchTimeWarper>(
187 t0, t1, rateStart, rateEnd);
188 else if(rateSlideType == SlideGeometricInput)
189 return std::make_unique<GeometricInputTimeWarper>(
190 t0, t1, rateStart, rateEnd);
191 else if(rateSlideType == SlideGeometricOutput)
192 return std::make_unique<GeometricOutputTimeWarper>(
193 t0, t1, rateStart, rateEnd);
194 else
195 return std::make_unique<IdentityTimeWarper>();
196}
197
199{
200 return EffectTypeProcess;
201}
202
203// Labels inside the affected region are moved to match the audio; labels after
204// it are shifted along appropriately.
206{
207 auto warper1 = createTimeWarper(
209 RegionTimeWarper warper{ mT0, mT1, std::move(warper1) };
210 lt->WarpLabels(warper);
211 return true;
212}
213
214double EffectSBSMS::getInvertedStretchedTime(double rateStart, double rateEnd, SlideType slideType, double outputTime)
215{
216 Slide slide(slideType,rateStart,rateEnd,0);
217 return slide.getInverseStretchedTime(outputTime);
218}
219
220double EffectSBSMS::getRate(double rateStart, double rateEnd, SlideType slideType, double t)
221{
222 Slide slide(slideType,rateStart,rateEnd,0);
223 return slide.getRate(t);
224}
225
227{
228 bool bGoodResult = true;
229
230 //Iterate over each track
231 //all needed because this effect needs to introduce silence in the group tracks to keep sync
232 EffectOutputTracks outputs { *mTracks, GetType(), { { mT0, mT1 } }, true };
233 mCurTrackNum = 0;
234
235 double maxDuration = 0.0;
236
237 Slide rateSlide(rateSlideType,rateStart,rateEnd);
238 Slide pitchSlide(pitchSlideType,pitchStart,pitchEnd);
239 mTotalStretch = rateSlide.getTotalStretch();
240
241 outputs.Get().Any().VisitWhile(bGoodResult,
242 [&](auto &&fallthrough){ return [&](LabelTrack &lt) {
243 if (!(lt.GetSelected() || SyncLock::IsSyncLockSelected(&lt)))
244 return fallthrough();
245 if (!ProcessLabelTrack(&lt))
246 bGoodResult = false;
247 }; },
248 [&](auto &&fallthrough){ return [&](WaveTrack &track) {
249 if (!track.GetSelected())
250 return fallthrough();
251
252 // Process only if the right marker is to the right of the left marker
253 if (mT1 > mT0) {
254 const auto start = track.TimeToLongSamples(mT0);
255 const auto end = track.TimeToLongSamples(mT1);
256
257 // TODO: more-than-two-channels
258 auto channels = track.Channels();
259 const auto leftTrack = (*channels.begin()).get();
260 const auto rightTrack = (channels.size() > 1)
261 ? (* ++ channels.first).get()
262 : nullptr;
263 if (rightTrack)
264 mCurTrackNum++; // Increment for rightTrack, too.
265
266 // SBSMS has a fixed sample rate - we just convert to its sample
267 // rate and then convert back
268 const float srTrack = track.GetRate();
269 const float srProcess = bLinkRatePitch ? srTrack : 44100.0;
270
271 // the resampler needs a callback to supply its samples
272 ResampleBuf rb;
273 const auto maxBlockSize = track.GetMaxBlockSize();
275 rb.buf.reinit(rb.blockSize, true);
276 rb.leftTrack = leftTrack;
277 rb.rightTrack = rightTrack ? rightTrack : leftTrack;
280
281 // Samples in selection
282 const auto samplesIn = end - start;
283
284 // Samples for SBSMS to process after resampling
285 const auto samplesToProcess = static_cast<sampleCount>(
286 samplesIn.as_float() * (srProcess/srTrack));
287
288 SlideType outSlideType;
289 SBSMSResampleCB outResampleCB;
290
291 if (bLinkRatePitch) {
292 rb.bPitch = true;
293 outSlideType = rateSlideType;
294 outResampleCB = resampleCB;
295 rb.offset = start;
296 rb.end = end;
297 // Third party library has its own type alias, check it
298 static_assert(sizeof(sampleCount::type) <=
299 sizeof(_sbsms_::SampleCountType),
300"Type _sbsms_::SampleCountType is too narrow to hold a sampleCount");
301 rb.iface = std::make_unique<SBSMSInterfaceSliding>(
302 &rateSlide, &pitchSlide, bPitchReferenceInput,
303 static_cast<_sbsms_::SampleCountType>(
304 samplesToProcess.as_long_long()),
305 0, nullptr);
306 }
307 else {
308 rb.bPitch = false;
309 outSlideType =
310 (srProcess == srTrack ? SlideIdentity : SlideConstant);
311 outResampleCB = postResampleCB;
312 rb.ratio = srProcess/srTrack;
313 rb.quality = std::make_unique<SBSMSQuality>(&SBSMSQualityStandard);
314 rb.resampler = std::make_unique<Resampler>(resampleCB, &rb,
315 srProcess == srTrack ? SlideIdentity : SlideConstant);
316 rb.sbsms = std::make_unique<SBSMS>(
317 rightTrack ? 2 : 1, rb.quality.get(), true);
318 rb.SBSMSBlockSize = rb.sbsms->getInputFrameSize();
319 rb.SBSMSBuf.reinit(static_cast<size_t>(rb.SBSMSBlockSize), true);
320 rb.offset = start;
321 rb.end = end;
322 rb.iface = std::make_unique<SBSMSEffectInterface>(
323 rb.resampler.get(), &rateSlide, &pitchSlide,
325 static_cast<_sbsms_::SampleCountType>(
326 samplesToProcess.as_long_long()),
327 0,
328 rb.quality.get());
329 }
330
331 Resampler resampler(outResampleCB, &rb, outSlideType);
332
333 audio outBuf[SBSMSOutBlockSize];
334 float outBufLeft[2 * SBSMSOutBlockSize];
335 float outBufRight[2 * SBSMSOutBlockSize];
336
337 // Samples in output after SBSMS
338 const sampleCount samplesToOutput = rb.iface->getSamplesToOutput();
339
340 // Samples in output after resampling back
341 const auto samplesOut = static_cast<sampleCount>(
342 samplesToOutput.as_float() * (srTrack / srProcess));
343
344 // Duration in track time
345 const double duration = (mT1 - mT0) * mTotalStretch;
346
347 if (duration > maxDuration)
348 maxDuration = duration;
349
350 const auto warper = createTimeWarper(
351 mT0, mT1, maxDuration, rateStart, rateEnd, rateSlideType);
352
353 std::shared_ptr<TrackList> tempList = track.WideEmptyCopy();
354 const auto outputTrack = *tempList->Any<WaveTrack>().begin();
355 auto iter = outputTrack->Channels().begin();
356 rb.outputTrack = outputTrack;
357 rb.outputLeftChannel = (*iter++).get();
358 if (rightTrack)
359 rb.outputRightChannel = (*iter).get();
360
361 long pos = 0;
362 long outputCount = -1;
363
364 // process
365 while (pos < samplesOut && outputCount) {
366 const auto frames =
367 limitSampleBufferSize(SBSMSOutBlockSize, samplesOut - pos);
368
369 outputCount = resampler.read(outBuf, frames);
370 for (int i = 0; i < outputCount; ++i) {
371 outBufLeft[i] = outBuf[i][0];
372 if (rightTrack)
373 outBufRight[i] = outBuf[i][1];
374 }
375 pos += outputCount;
377 (samplePtr)outBufLeft, floatSample, outputCount);
378 if (rightTrack)
380 (samplePtr)outBufRight, floatSample, outputCount);
381
382 double frac = static_cast<double>(pos) / samplesOut.as_double();
383 int nWhichTrack = mCurTrackNum;
384 if (rightTrack) {
385 nWhichTrack = 2 * (mCurTrackNum / 2);
386 if (frac < 0.5)
387 // Show twice as far for each track,
388 // because we're doing 2 at once.
389 frac *= 2.0;
390 else {
391 nWhichTrack++;
392 frac -= 0.5;
393 // Show twice as far for each track,
394 // because we're doing 2 at once.
395 frac *= 2.0;
396 }
397 }
398 if (TrackProgress(nWhichTrack, frac)) {
399 bGoodResult = false;
400 return;
401 }
402 }
403
404 {
405 auto pException = rb.mpException;
406 rb.mpException = {};
407 if (pException)
408 std::rethrow_exception(pException);
409 }
410
411 rb.outputTrack->Flush();
412 Finalize(track, *outputTrack, *warper);
413 }
414 mCurTrackNum++;
415 }; },
416 [&](Track &t) {
417 // Outer loop is over leaders, so fall-through must check for
418 // multiple channels
420 t.SyncLockAdjust(mT1, mT0 + (mT1 - mT0) * mTotalStretch);
421 }
422 );
423
424 if (bGoodResult)
425 outputs.Commit();
426
427 return bGoodResult;
428}
429
431 WaveTrack &orig, const WaveTrack &out, const TimeWarper &warper)
432{
433 assert(orig.IsLeader());
434 assert(out.IsLeader());
435 assert(orig.NChannels() == out.NChannels());
436 // Silenced samples will be inserted in gaps between clips, so capture where these
437 // gaps are for later deletion
438 std::vector<std::pair<double, double>> gaps;
439 double last = mT0;
440 auto clips = orig.SortedClipArray();
441 auto front = clips.front();
442 auto back = clips.back();
443 for (auto &clip : clips) {
444 auto st = clip->GetPlayStartTime();
445 auto et = clip->GetPlayEndTime();
446
447 if (st >= mT0 || et < mT1) {
448 if (mT0 < st && clip == front) {
449 gaps.push_back(std::make_pair(mT0, st));
450 }
451 else if (last < st && mT0 <= last ) {
452 gaps.push_back(std::make_pair(last, st));
453 }
454
455 if (et < mT1 && clip == back) {
456 gaps.push_back(std::make_pair(et, mT1));
457 }
458 }
459 last = et;
460 }
461
462 // Take the output track and insert it in place of the original sample data
463 orig.ClearAndPaste(mT0, mT1, out, true, true, &warper);
464
465 // Finally, recreate the gaps
466 for (auto gap : gaps) {
467 const auto st = orig.SnapToSample(gap.first);
468 const auto et = orig.SnapToSample(gap.second);
469 if (st >= mT0 && et <= mT1 && st != et)
470 orig.SplitDelete(warper.Warp(st), warper.Warp(et));
471 }
472}
473
474#endif
EffectType
@ EffectTypeProcess
static const int gap
Definition: MeterPanel.cpp:257
std::unique_ptr< TimeWarper > createTimeWarper(double t0, double t1, double duration, double rateStart, double rateEnd, SlideType rateSlideType)
@ SBSMSOutBlockSize
Definition: SBSMSEffect.cpp:28
long postResampleCB(void *cb_data, SBSMSFrame *data)
long resampleCB(void *cb_data, SBSMSFrame *data)
Definition: SBSMSEffect.cpp:90
size_t limitSampleBufferSize(size_t bufferSize, sampleCount limit)
Definition: SampleCount.cpp:22
char * samplePtr
Definition: SampleFormat.h:57
MockedAudio audio
Contains declarations for TimeWarper, IdentityTimeWarper, ShiftTimeWarper, LinearTimeWarper,...
void reinit(Integral count, bool initialize=false)
Definition: MemoryX.h:56
double mT1
Definition: EffectBase.h:116
std::shared_ptr< TrackList > mTracks
Definition: EffectBase.h:109
double mT0
Definition: EffectBase.h:115
bool TrackProgress(int whichTrack, double frac, const TranslatableString &={}) const
Definition: Effect.cpp:343
Performs effect computation.
Use this object to copy the input tracks to tentative outputTracks.
SlideType rateSlideType
Definition: SBSMSEffect.h:61
bool bLinkRatePitch
Definition: SBSMSEffect.h:60
void setParameters(double rateStart, double rateEnd, double pitchStart, double pitchEnd, SlideType rateSlideType, SlideType pitchSlideType, bool bLinkRatePitch, bool bRateReferenceInput, bool bPitchReferenceInput)
double pitchStart
Definition: SBSMSEffect.h:59
bool ProcessLabelTrack(LabelTrack *track)
int mCurTrackNum
Definition: SBSMSEffect.h:63
bool bPitchReferenceInput
Definition: SBSMSEffect.h:60
float mTotalStretch
Definition: SBSMSEffect.h:64
static double getInvertedStretchedTime(double rateStart, double rateEnd, SlideType slideType, double outputTime)
EffectType GetType() const override
Type determines how it behaves.
double rateStart
Definition: SBSMSEffect.h:59
SlideType pitchSlideType
Definition: SBSMSEffect.h:62
double pitchEnd
Definition: SBSMSEffect.h:59
bool bRateReferenceInput
Definition: SBSMSEffect.h:60
double rateEnd
Definition: SBSMSEffect.h:59
bool Process(EffectInstance &instance, EffectSettings &settings) override
void Finalize(WaveTrack &orig, const WaveTrack &out, const TimeWarper &warper)
static double getRate(double rateStart, double rateEnd, SlideType slideType, double t)
A LabelTrack is a Track that holds labels (LabelStruct).
Definition: LabelTrack.h:87
void WarpLabels(const TimeWarper &warper)
Definition: LabelTrack.cpp:294
No change before the specified region; during the region, warp according to the given warper; after t...
Definition: TimeWarper.h:192
ArrayOf< audio > SBSMSBuf
Definition: SBSMSEffect.cpp:57
sampleCount offset
Definition: SBSMSEffect.cpp:49
WaveChannel * outputRightChannel
Definition: SBSMSEffect.cpp:64
std::unique_ptr< SBSMSInterface > iface
Definition: SBSMSEffect.cpp:56
ArrayOf< audio > buf
Definition: SBSMSEffect.cpp:44
ArrayOf< float > leftBuffer
Definition: SBSMSEffect.cpp:51
long SBSMSBlockSize
Definition: SBSMSEffect.cpp:48
std::unique_ptr< SBSMSQuality > quality
Definition: SBSMSEffect.cpp:61
sampleCount processed
Definition: SBSMSEffect.cpp:46
WaveChannel * outputLeftChannel
Definition: SBSMSEffect.cpp:63
std::exception_ptr mpException
Definition: SBSMSEffect.cpp:66
WaveChannel * leftTrack
Definition: SBSMSEffect.cpp:53
WaveChannel * rightTrack
Definition: SBSMSEffect.cpp:54
ArrayOf< float > rightBuffer
Definition: SBSMSEffect.cpp:52
double ratio
Definition: SBSMSEffect.cpp:45
std::unique_ptr< SBSMS > sbsms
Definition: SBSMSEffect.cpp:55
std::unique_ptr< Resampler > resampler
Definition: SBSMSEffect.cpp:60
size_t blockSize
Definition: SBSMSEffect.cpp:47
sampleCount end
Definition: SBSMSEffect.cpp:50
WaveTrack * outputTrack
Definition: SBSMSEffect.cpp:62
long samples(audio *buf, long n)
Definition: SBSMSEffect.cpp:82
Resampler * resampler
Definition: SBSMSEffect.cpp:87
virtual ~SBSMSEffectInterface()
Definition: SBSMSEffect.cpp:80
SBSMSEffectInterface(Resampler *resampler, Slide *rateSlide, Slide *pitchSlide, bool bReferenceInput, const SampleCountType samples, long preSamples, SBSMSQuality *quality)
Definition: SBSMSEffect.cpp:71
static bool IsSyncLockSelected(const Track *pTrack)
Definition: SyncLock.cpp:82
Transforms one point in time to another point. For example, a time stretching effect might use one to...
Definition: TimeWarper.h:62
virtual double Warp(double originalTime) const =0
Abstract base class for an object holding data associated with points on a time axis.
Definition: Track.h:122
bool Append(constSamplePtr buffer, sampleFormat format, size_t len)
Definition: WaveTrack.cpp:2711
bool GetFloats(float *buffer, sampleCount start, size_t len, fillFormat fill=FillFormat::fillZero, bool mayThrow=true, sampleCount *pNumWithinClips=nullptr) const
"narrow" overload fetches from the unique channel
Definition: WaveTrack.h:159
size_t GetBestBlockSize(sampleCount t) const
A hint for sizing of well aligned fetches.
Definition: WaveTrack.h:1246
A Track that contains audio waveform data.
Definition: WaveTrack.h:227
void SplitDelete(double t0, double t1)
Definition: WaveTrack.cpp:1891
WaveClipPointers SortedClipArray()
Definition: WaveTrack.cpp:4425
bool IsLeader() const override
Definition: WaveTrack.cpp:2833
void Flush() override
Definition: WaveTrack.cpp:2789
size_t NChannels() const override
May report more than one only when this is a leader track.
Definition: WaveTrack.cpp:849
void ClearAndPaste(double t0, double t1, const WaveTrack &src, bool preserve=true, bool merge=true, const TimeWarper *effectWarper=nullptr, bool clearByTrimming=false)
Definition: WaveTrack.cpp:1554
double SnapToSample(double t) const
Positions or offsets within audio files need a wide type.
Definition: SampleCount.h:19
long long type
Definition: SampleCount.h:21
float as_float() const
Definition: SampleCount.h:45
auto end(const Ptr< Type, BaseDeleter > &p)
Enables range-for.
Definition: PackedArray.h:159
auto begin(const Ptr< Type, BaseDeleter > &p)
Enables range-for.
Definition: PackedArray.h:150
Externalized state of a plug-in.