Audacity 3.2.0
SBSMSEffect.cpp
Go to the documentation of this file.
1/**********************************************************************
2
3Audacity: A Digital Audio Editor
4
5SBSMSEffect.cpp
6
7Clayton Otey
8
9This class contains all of the common code for an
10effect that uses SBSMS to do its processing (TimeScale)
11
12**********************************************************************/
13#if USE_SBSMS
14#include "SBSMSEffect.h"
15#include "EffectOutputTracks.h"
16
17#include <math.h>
18
19#include "../LabelTrack.h"
20#include "SyncLock.h"
21#include "WaveClip.h"
22#include "WaveTrack.h"
23#include "TimeWarper.h"
24
25#include <cassert>
26
27enum {
29};
30
32{
33public:
35 {
36 processed = 0;
37 }
38
40 {
41 }
42
43 bool bPitch;
45 double ratio;
47 size_t blockSize;
55 std::unique_ptr<SBSMS> sbsms;
56 std::unique_ptr<SBSMSInterface> iface;
58
59 // Not required by callbacks, but makes for easier cleanup
60 std::unique_ptr<Resampler> resampler;
61 std::unique_ptr<SBSMSQuality> quality;
65
66 std::exception_ptr mpException {};
67};
68
69class SBSMSEffectInterface final : public SBSMSInterfaceSliding {
70public:
72 Slide *rateSlide, Slide *pitchSlide,
73 bool bReferenceInput,
74 const SampleCountType samples, long preSamples,
75 SBSMSQuality *quality)
76 : SBSMSInterfaceSliding(rateSlide,pitchSlide,bReferenceInput,samples,preSamples,quality)
77 {
78 this->resampler = resampler;
79 }
81
82 long samples(audio *buf, long n) {
83 return resampler->read(buf, n);
84 }
85
86protected:
87 Resampler *resampler;
88};
89
90long resampleCB(void *cb_data, SBSMSFrame *data)
91{
92 ResampleBuf *r = (ResampleBuf*) cb_data;
93
94 auto blockSize = limitSampleBufferSize(
96 r->end - r->offset
97 );
98
99 // Get the samples from the tracks and put them in the buffers.
100 // I don't know if we can safely propagate errors through sbsms, and it
101 // does not seem to let us report error codes, so use this roundabout to
102 // stop the effect early.
103 try {
105 (r->leftBuffer.get()), r->offset, blockSize);
107 (r->rightBuffer.get()), r->offset, blockSize);
108 }
109 catch ( ... ) {
110 // Save the exception object for re-throw when out of the library
111 r->mpException = std::current_exception();
112 data->size = 0;
113 return 0;
114 }
115
116 // convert to sbsms audio format
117 for(decltype(blockSize) i=0; i<blockSize; i++) {
118 r->buf[i][0] = r->leftBuffer[i];
119 r->buf[i][1] = r->rightBuffer[i];
120 }
121
122 data->buf = r->buf.get();
123 data->size = blockSize;
124 if(r->bPitch) {
125 float t0 = r->processed.as_float() / r->iface->getSamplesToInput();
126 float t1 = (r->processed + blockSize).as_float() / r->iface->getSamplesToInput();
127 data->ratio0 = r->iface->getStretch(t0);
128 data->ratio1 = r->iface->getStretch(t1);
129 } else {
130 data->ratio0 = r->ratio;
131 data->ratio1 = r->ratio;
132 }
133 r->processed += blockSize;
134 r->offset += blockSize;
135 return blockSize;
136}
137
138long postResampleCB(void *cb_data, SBSMSFrame *data)
139{
140 ResampleBuf *r = (ResampleBuf*) cb_data;
141 auto count = r->sbsms->read(r->iface.get(), r->SBSMSBuf.get(), r->SBSMSBlockSize);
142 data->buf = r->SBSMSBuf.get();
143 data->size = count;
144 data->ratio0 = 1.0 / r->ratio;
145 data->ratio1 = 1.0 / r->ratio;
146 return count;
147}
148
149void EffectSBSMS :: setParameters(double rateStartIn, double rateEndIn, double pitchStartIn, double pitchEndIn,
150 SlideType rateSlideTypeIn, SlideType pitchSlideTypeIn,
151 bool bLinkRatePitchIn, bool bRateReferenceInputIn, bool bPitchReferenceInputIn)
152{
153 this->rateStart = rateStartIn;
154 this->rateEnd = rateEndIn;
155 this->pitchStart = pitchStartIn;
156 this->pitchEnd = pitchEndIn;
157 this->bLinkRatePitch = bLinkRatePitchIn;
158 this->rateSlideType = rateSlideTypeIn;
159 this->pitchSlideType = pitchSlideTypeIn;
160 this->bRateReferenceInput = bRateReferenceInputIn;
161 this->bPitchReferenceInput = bPitchReferenceInputIn;
162}
163
164void EffectSBSMS::setParameters(double tempoRatio, double pitchRatio)
165{
166 setParameters(tempoRatio, tempoRatio, pitchRatio, pitchRatio,
167 SlideConstant, SlideConstant, false, false, false);
168}
169
170std::unique_ptr<TimeWarper> createTimeWarper(double t0, double t1, double duration,
171 double rateStart, double rateEnd, SlideType rateSlideType)
172{
173 if (rateStart == rateEnd || rateSlideType == SlideConstant)
174 return std::make_unique<LinearTimeWarper>(
175 t0, t0, t1, t0 + duration);
176 else if(rateSlideType == SlideLinearInputRate)
177 return std::make_unique<LinearInputRateTimeWarper>(
178 t0, t1, rateStart, rateEnd);
179 else if(rateSlideType == SlideLinearOutputRate)
180 return std::make_unique<LinearOutputRateTimeWarper>(
181 t0, t1, rateStart, rateEnd);
182 else if(rateSlideType == SlideLinearInputStretch)
183 return std::make_unique<LinearInputStretchTimeWarper>(
184 t0, t1, rateStart, rateEnd);
185 else if(rateSlideType == SlideLinearOutputStretch)
186 return std::make_unique<LinearOutputStretchTimeWarper>(
187 t0, t1, rateStart, rateEnd);
188 else if(rateSlideType == SlideGeometricInput)
189 return std::make_unique<GeometricInputTimeWarper>(
190 t0, t1, rateStart, rateEnd);
191 else if(rateSlideType == SlideGeometricOutput)
192 return std::make_unique<GeometricOutputTimeWarper>(
193 t0, t1, rateStart, rateEnd);
194 else
195 return std::make_unique<IdentityTimeWarper>();
196}
197
199{
200 return EffectTypeProcess;
201}
202
203// Labels inside the affected region are moved to match the audio; labels after
204// it are shifted along appropriately.
206{
207 auto warper1 = createTimeWarper(
209 RegionTimeWarper warper{ mT0, mT1, std::move(warper1) };
210 lt->WarpLabels(warper);
211 return true;
212}
213
214double EffectSBSMS::getInvertedStretchedTime(double rateStart, double rateEnd, SlideType slideType, double outputTime)
215{
216 Slide slide(slideType,rateStart,rateEnd,0);
217 return slide.getInverseStretchedTime(outputTime);
218}
219
220double EffectSBSMS::getRate(double rateStart, double rateEnd, SlideType slideType, double t)
221{
222 Slide slide(slideType,rateStart,rateEnd,0);
223 return slide.getRate(t);
224}
225
227{
228 bool bGoodResult = true;
229
230 //Iterate over each track
231 //all needed because this effect needs to introduce silence in the group tracks to keep sync
232 EffectOutputTracks outputs { *mTracks, GetType(), { { mT0, mT1 } }, true };
233 mCurTrackNum = 0;
234
235 double maxDuration = 0.0;
236
237 Slide rateSlide(rateSlideType,rateStart,rateEnd);
238 Slide pitchSlide(pitchSlideType,pitchStart,pitchEnd);
239 mTotalStretch = rateSlide.getTotalStretch();
240
241 outputs.Get().Any().VisitWhile(bGoodResult,
242 [&](auto &&fallthrough){ return [&](LabelTrack &lt) {
243 if (!(lt.GetSelected() || SyncLock::IsSyncLockSelected(lt)))
244 return fallthrough();
245 if (!ProcessLabelTrack(&lt))
246 bGoodResult = false;
247 }; },
248 [&](auto &&fallthrough){ return [&](WaveTrack &track) {
249 if (!track.GetSelected())
250 return fallthrough();
251
252 // Process only if the right marker is to the right of the left marker
253 if (mT1 > mT0) {
254 const auto start = track.TimeToLongSamples(mT0);
255 const auto end = track.TimeToLongSamples(mT1);
256
257 // TODO: more-than-two-channels
258 auto channels = track.Channels();
259 const auto leftTrack = (*channels.begin()).get();
260 const auto rightTrack = (channels.size() > 1)
261 ? (* ++ channels.first).get()
262 : nullptr;
263 if (rightTrack)
264 mCurTrackNum++; // Increment for rightTrack, too.
265
266 // SBSMS has a fixed sample rate - we just convert to its sample
267 // rate and then convert back
268 const float srTrack = track.GetRate();
269 const float srProcess = bLinkRatePitch ? srTrack : 44100.0;
270
271 // the resampler needs a callback to supply its samples
272 ResampleBuf rb;
273 const auto maxBlockSize = track.GetMaxBlockSize();
275 rb.buf.reinit(rb.blockSize, true);
276 rb.leftTrack = leftTrack;
277 rb.rightTrack = rightTrack ? rightTrack : leftTrack;
280
281 // Samples in selection
282 const auto samplesIn = end - start;
283
284 // Samples for SBSMS to process after resampling
285 const auto samplesToProcess = static_cast<sampleCount>(
286 samplesIn.as_float() * (srProcess/srTrack));
287
288 SlideType outSlideType;
289 SBSMSResampleCB outResampleCB;
290
291 if (bLinkRatePitch) {
292 rb.bPitch = true;
293 outSlideType = rateSlideType;
294 outResampleCB = resampleCB;
295 rb.offset = start;
296 rb.end = end;
297 // Third party library has its own type alias, check it
298 static_assert(sizeof(sampleCount::type) <=
299 sizeof(_sbsms_::SampleCountType),
300"Type _sbsms_::SampleCountType is too narrow to hold a sampleCount");
301 rb.iface = std::make_unique<SBSMSInterfaceSliding>(
302 &rateSlide, &pitchSlide, bPitchReferenceInput,
303 static_cast<_sbsms_::SampleCountType>(
304 samplesToProcess.as_long_long()),
305 0, nullptr);
306 }
307 else {
308 rb.bPitch = false;
309 outSlideType =
310 (srProcess == srTrack ? SlideIdentity : SlideConstant);
311 outResampleCB = postResampleCB;
312 rb.ratio = srProcess/srTrack;
313 rb.quality = std::make_unique<SBSMSQuality>(&SBSMSQualityStandard);
314 rb.resampler = std::make_unique<Resampler>(resampleCB, &rb,
315 srProcess == srTrack ? SlideIdentity : SlideConstant);
316 rb.sbsms = std::make_unique<SBSMS>(
317 rightTrack ? 2 : 1, rb.quality.get(), true);
318 rb.SBSMSBlockSize = rb.sbsms->getInputFrameSize();
319 rb.SBSMSBuf.reinit(static_cast<size_t>(rb.SBSMSBlockSize), true);
320 rb.offset = start;
321 rb.end = end;
322 rb.iface = std::make_unique<SBSMSEffectInterface>(
323 rb.resampler.get(), &rateSlide, &pitchSlide,
325 static_cast<_sbsms_::SampleCountType>(
326 samplesToProcess.as_long_long()),
327 0,
328 rb.quality.get());
329 }
330
331 Resampler resampler(outResampleCB, &rb, outSlideType);
332
333 audio outBuf[SBSMSOutBlockSize];
334 float outBufLeft[2 * SBSMSOutBlockSize];
335 float outBufRight[2 * SBSMSOutBlockSize];
336
337 // Samples in output after SBSMS
338 const sampleCount samplesToOutput = rb.iface->getSamplesToOutput();
339
340 // Samples in output after resampling back
341 const auto samplesOut = static_cast<sampleCount>(
342 samplesToOutput.as_float() * (srTrack / srProcess));
343
344 // Duration in track time
345 const double duration = (mT1 - mT0) * mTotalStretch;
346
347 if (duration > maxDuration)
348 maxDuration = duration;
349
350 const auto warper = createTimeWarper(
351 mT0, mT1, maxDuration, rateStart, rateEnd, rateSlideType);
352
353 WaveTrack::Holder outputTrack = track.EmptyCopy();
354 auto iter = outputTrack->Channels().begin();
355 rb.outputTrack = outputTrack.get();
356 rb.outputLeftChannel = (*iter++).get();
357 if (rightTrack)
358 rb.outputRightChannel = (*iter).get();
359
360 long pos = 0;
361 long outputCount = -1;
362
363 // process
364 while (pos < samplesOut && outputCount) {
365 const auto frames =
366 limitSampleBufferSize(SBSMSOutBlockSize, samplesOut - pos);
367
368 outputCount = resampler.read(outBuf, frames);
369 for (int i = 0; i < outputCount; ++i) {
370 outBufLeft[i] = outBuf[i][0];
371 if (rightTrack)
372 outBufRight[i] = outBuf[i][1];
373 }
374 pos += outputCount;
376 (samplePtr)outBufLeft, floatSample, outputCount);
377 if (rightTrack)
379 (samplePtr)outBufRight, floatSample, outputCount);
380
381 double frac = static_cast<double>(pos) / samplesOut.as_double();
382 int nWhichTrack = mCurTrackNum;
383 if (rightTrack) {
384 nWhichTrack = 2 * (mCurTrackNum / 2);
385 if (frac < 0.5)
386 // Show twice as far for each track,
387 // because we're doing 2 at once.
388 frac *= 2.0;
389 else {
390 nWhichTrack++;
391 frac -= 0.5;
392 // Show twice as far for each track,
393 // because we're doing 2 at once.
394 frac *= 2.0;
395 }
396 }
397 if (TrackProgress(nWhichTrack, frac)) {
398 bGoodResult = false;
399 return;
400 }
401 }
402
403 {
404 auto pException = rb.mpException;
405 rb.mpException = {};
406 if (pException)
407 std::rethrow_exception(pException);
408 }
409
410 rb.outputTrack->Flush();
411 Finalize(track, *outputTrack, *warper);
412 }
413 mCurTrackNum++;
414 }; },
415 [&](Track &t) {
417 t.SyncLockAdjust(mT1, mT0 + (mT1 - mT0) * mTotalStretch);
418 }
419 );
420
421 if (bGoodResult)
422 outputs.Commit();
423
424 return bGoodResult;
425}
426
428 WaveTrack &orig, const WaveTrack &out, const TimeWarper &warper)
429{
430 assert(orig.NChannels() == out.NChannels());
431 // Silenced samples will be inserted in gaps between clips, so capture where these
432 // gaps are for later deletion
433 std::vector<std::pair<double, double>> gaps;
434 double last = mT0;
435 auto clips = orig.SortedIntervalArray();
436 auto front = clips.front();
437 auto back = clips.back();
438 for (auto &clip : clips) {
439 auto st = clip->GetPlayStartTime();
440 auto et = clip->GetPlayEndTime();
441
442 if (st >= mT0 || et < mT1) {
443 if (mT0 < st && clip == front) {
444 gaps.push_back(std::make_pair(mT0, st));
445 }
446 else if (last < st && mT0 <= last ) {
447 gaps.push_back(std::make_pair(last, st));
448 }
449
450 if (et < mT1 && clip == back) {
451 gaps.push_back(std::make_pair(et, mT1));
452 }
453 }
454 last = et;
455 }
456
457 // Take the output track and insert it in place of the original sample data
458 orig.ClearAndPaste(mT0, mT1, out, true, true, &warper);
459
460 // Finally, recreate the gaps
461 for (auto gap : gaps) {
462 const auto st = orig.SnapToSample(gap.first);
463 const auto et = orig.SnapToSample(gap.second);
464 if (st >= mT0 && et <= mT1 && st != et)
465 orig.SplitDelete(warper.Warp(st), warper.Warp(et));
466 }
467}
468
469#endif
EffectType
@ EffectTypeProcess
static const int gap
Definition: MeterPanel.cpp:257
@ SBSMSOutBlockSize
Definition: SBSMSEffect.cpp:28
std::unique_ptr< TimeWarper > createTimeWarper(double t0, double t1, double duration, double rateStart, double rateEnd, SlideType rateSlideType)
long postResampleCB(void *cb_data, SBSMSFrame *data)
long resampleCB(void *cb_data, SBSMSFrame *data)
Definition: SBSMSEffect.cpp:90
size_t limitSampleBufferSize(size_t bufferSize, sampleCount limit)
Definition: SampleCount.cpp:22
char * samplePtr
Definition: SampleFormat.h:57
MockedAudio audio
Contains declarations for TimeWarper, IdentityTimeWarper, ShiftTimeWarper, LinearTimeWarper,...
void reinit(Integral count, bool initialize=false)
Definition: MemoryX.h:59
double mT1
Definition: EffectBase.h:114
std::shared_ptr< TrackList > mTracks
Definition: EffectBase.h:107
double mT0
Definition: EffectBase.h:113
bool TrackProgress(int whichTrack, double frac, const TranslatableString &={}) const
Definition: Effect.cpp:343
Performs effect computation.
Use this object to copy the input tracks to tentative outputTracks.
SlideType rateSlideType
Definition: SBSMSEffect.h:59
bool bLinkRatePitch
Definition: SBSMSEffect.h:58
void setParameters(double rateStart, double rateEnd, double pitchStart, double pitchEnd, SlideType rateSlideType, SlideType pitchSlideType, bool bLinkRatePitch, bool bRateReferenceInput, bool bPitchReferenceInput)
double pitchStart
Definition: SBSMSEffect.h:57
bool ProcessLabelTrack(LabelTrack *track)
int mCurTrackNum
Definition: SBSMSEffect.h:61
bool bPitchReferenceInput
Definition: SBSMSEffect.h:58
float mTotalStretch
Definition: SBSMSEffect.h:62
static double getInvertedStretchedTime(double rateStart, double rateEnd, SlideType slideType, double outputTime)
EffectType GetType() const override
Type determines how it behaves.
double rateStart
Definition: SBSMSEffect.h:57
SlideType pitchSlideType
Definition: SBSMSEffect.h:60
double pitchEnd
Definition: SBSMSEffect.h:57
bool bRateReferenceInput
Definition: SBSMSEffect.h:58
double rateEnd
Definition: SBSMSEffect.h:57
bool Process(EffectInstance &instance, EffectSettings &settings) override
void Finalize(WaveTrack &orig, const WaveTrack &out, const TimeWarper &warper)
static double getRate(double rateStart, double rateEnd, SlideType slideType, double t)
A LabelTrack is a Track that holds labels (LabelStruct).
Definition: LabelTrack.h:95
void WarpLabels(const TimeWarper &warper)
Definition: LabelTrack.cpp:297
No change before the specified region; during the region, warp according to the given warper; after t...
Definition: TimeWarper.h:192
ArrayOf< audio > SBSMSBuf
Definition: SBSMSEffect.cpp:57
sampleCount offset
Definition: SBSMSEffect.cpp:49
WaveChannel * outputRightChannel
Definition: SBSMSEffect.cpp:64
std::unique_ptr< SBSMSInterface > iface
Definition: SBSMSEffect.cpp:56
ArrayOf< audio > buf
Definition: SBSMSEffect.cpp:44
ArrayOf< float > leftBuffer
Definition: SBSMSEffect.cpp:51
long SBSMSBlockSize
Definition: SBSMSEffect.cpp:48
std::unique_ptr< SBSMSQuality > quality
Definition: SBSMSEffect.cpp:61
sampleCount processed
Definition: SBSMSEffect.cpp:46
WaveChannel * outputLeftChannel
Definition: SBSMSEffect.cpp:63
std::exception_ptr mpException
Definition: SBSMSEffect.cpp:66
WaveChannel * leftTrack
Definition: SBSMSEffect.cpp:53
WaveChannel * rightTrack
Definition: SBSMSEffect.cpp:54
ArrayOf< float > rightBuffer
Definition: SBSMSEffect.cpp:52
double ratio
Definition: SBSMSEffect.cpp:45
std::unique_ptr< SBSMS > sbsms
Definition: SBSMSEffect.cpp:55
std::unique_ptr< Resampler > resampler
Definition: SBSMSEffect.cpp:60
size_t blockSize
Definition: SBSMSEffect.cpp:47
sampleCount end
Definition: SBSMSEffect.cpp:50
WaveTrack * outputTrack
Definition: SBSMSEffect.cpp:62
long samples(audio *buf, long n)
Definition: SBSMSEffect.cpp:82
Resampler * resampler
Definition: SBSMSEffect.cpp:87
virtual ~SBSMSEffectInterface()
Definition: SBSMSEffect.cpp:80
SBSMSEffectInterface(Resampler *resampler, Slide *rateSlide, Slide *pitchSlide, bool bReferenceInput, const SampleCountType samples, long preSamples, SBSMSQuality *quality)
Definition: SBSMSEffect.cpp:71
static bool IsSyncLockSelected(const Track &track)
Definition: SyncLock.cpp:80
Transforms one point in time to another point. For example, a time stretching effect might use one to...
Definition: TimeWarper.h:62
virtual double Warp(double originalTime) const =0
Abstract base class for an object holding data associated with points on a time axis.
Definition: Track.h:110
bool Append(constSamplePtr buffer, sampleFormat format, size_t len)
Definition: WaveTrack.cpp:2216
bool GetFloats(float *buffer, sampleCount start, size_t len, fillFormat fill=FillFormat::fillZero, bool mayThrow=true, sampleCount *pNumWithinClips=nullptr) const
"narrow" overload fetches from the unique channel
Definition: WaveTrack.h:129
size_t GetBestBlockSize(sampleCount t) const
A hint for sizing of well aligned fetches.
Definition: WaveTrack.h:850
A Track that contains audio waveform data.
Definition: WaveTrack.h:203
void SplitDelete(double t0, double t1)
Definition: WaveTrack.cpp:1503
void Flush() override
Definition: WaveTrack.cpp:2294
IntervalHolders SortedIntervalArray()
Return all WaveClips sorted by clip play start time.
Definition: WaveTrack.cpp:3270
size_t NChannels() const override
A constant property.
Definition: WaveTrack.cpp:532
std::shared_ptr< WaveTrack > Holder
Definition: WaveTrack.h:247
void ClearAndPaste(double t0, double t1, const WaveTrack &src, bool preserve=true, bool merge=true, const TimeWarper *effectWarper=nullptr, bool clearByTrimming=false)
Definition: WaveTrack.cpp:1196
double SnapToSample(double t) const
Positions or offsets within audio files need a wide type.
Definition: SampleCount.h:19
long long type
Definition: SampleCount.h:21
float as_float() const
Definition: SampleCount.h:45
const char * end(const char *str) noexcept
Definition: StringUtils.h:106
Externalized state of a plug-in.