kenmirrors/asterisk
1
0
Fork 0
mirror of https://github.com/asterisk/asterisk.git synced 2025-09-03 11:25:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Binaural synthesis (confbridge): Adds binaural synthesis to bridge_softmix.

Browse Source 
Adds binaural synthesis to bridge_softmix (via convolution using libfftw3).
Binaural synthesis is conducted at 48kHz.
For a conference, only one spatial representation is rendered.
The default rendering is applied for mono-capable channels.

ASTERISK-26292

Change-Id: Iecdb381b6adc17c961049658678f6219adae1ddf
This commit is contained in:
frahaase
2016-08-12 18:23:18 +02:00
committed by George Joseph
parent 9ad1df71b3
commit 094c26aa68
21 changed files with 52186 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

579
bridges/bridge_softmix/bridge_softmix_binaural.c Normal file
View File

@@ -0,0 +1,579 @@
/*
* Asterisk -- An open source telephony toolkit.
*
* Copyright (C) 2016, Frank Haase, Dennis Guse
*
* Frank Haase <fra.haase@gmail.com>
* Dennis Guse <dennis.guse@alumni.tu-berlin.de>
*
* See http://www.asterisk.org for more information about
* the Asterisk project. Please do not directly contact
* any of the maintainers of this project for assistance;
* the project provides a web site, mailing lists and IRC
* channels for your use.
*
* This program is free software, distributed under the terms of
* the GNU General Public License Version 2. See the LICENSE file
* at the top of the source tree.
*/
/*! \file
*
* \brief Multi-party software based binaural mixing
*
* \author Frank Haase <fra.haase@googlemail.com>
* \author Dennis Guse <dennis.guse@alumni.tu-berlin.de>
*
* \ingroup bridges
*/
#include "include/bridge_softmix_internal.h"
#ifdef BINAURAL_RENDERING
#include "include/hrirs_configuration.h"
#endif
/*! The number of prealloced channels when a bridge will be created. */
#define CONVOLVE_CHANNEL_PREALLOC 3
/*! Max size of the convolve buffer. */
#define CONVOLVE_MAX_BUFFER 4096
/*! The default sample size in an binaural environment with a two-channel
* codec at 48kHz is 960 samples.
*/
#define CONVOLUTION_SAMPLE_SIZE 960
#ifdef BINAURAL_RENDERING
#if SOFTMIX_BINAURAL_SAMPLE_RATE != HRIRS_SAMPLE_RATE
#error HRIRs are required to be SOFTMIX_BINAURAL_SAMPLE_RATE Hz. Please adjust hrirs.h accordingly.
#endif
#if CONVOLUTION_SAMPLE_SIZE < HRIRS_IMPULSE_LEN
#error HRIRS_IMPULSE_LEN cannot be longer than CONVOLUTION_SAMPLE_SIZE. Please adjust hrirs.h accordingly.
#endif
#endif
void reset_channel_pair(struct convolve_channel_pair *channel_pair,
unsigned int default_sample_size)
{
memset(channel_pair->chan_left.overlap_add, 0, sizeof(float) * default_sample_size);
memset(channel_pair->chan_right.overlap_add, 0, sizeof(float) * default_sample_size);
}
void random_binaural_pos_change(struct softmix_bridge_data *softmix_data)
{
/*
* We perform a shuffle of all channels, even the ones that aren't used at the
* moment of shuffling now. This has the efect that new members will be placed
* randomly too.
*/
unsigned int i;
unsigned int j;
struct convolve_channel_pair *tmp;
if (softmix_data->convolve.chan_size < 2) {
return;
}
srand(time(NULL));
for (i = softmix_data->convolve.chan_size - 1; i > 0; i--) {
j = rand() % (i + 1);
tmp = softmix_data->convolve.cchan_pair[i];
reset_channel_pair(tmp, softmix_data->default_sample_size);
softmix_data->convolve.cchan_pair[i] = softmix_data->convolve.cchan_pair[j];
softmix_data->convolve.cchan_pair[j] = tmp;
}
}
int do_convolve(struct convolve_channel *chan, int16_t *in_samples,
unsigned int in_sample_size, unsigned int hrtf_length)
{
#ifdef BINAURAL_RENDERING
unsigned int i;
if (in_sample_size != CONVOLUTION_SAMPLE_SIZE) {
return -1;
}
/* FFT setting real part */
for (i = 0; i < CONVOLUTION_SAMPLE_SIZE; i++) {
chan->fftw_in[i] = in_samples[i] * (FLT_MAX / SHRT_MAX);
}
for (i = CONVOLUTION_SAMPLE_SIZE; i < hrtf_length; i++) {
chan->fftw_in[i] = 0;
}
fftw_execute(chan->fftw_plan);
/* Imaginary mulitplication (frequency space). */
/* First FFTW result has never an imaginary part. */
chan->fftw_in[0] = chan->fftw_out[0] * chan->hrtf[0];
for (i = 1; i < (hrtf_length / 2); i++) {
/* Real part */
chan->fftw_in[i] = (chan->fftw_out[i] * chan->hrtf[i]) -
(chan->fftw_out[hrtf_length - i] * chan->hrtf[hrtf_length - i]);
/* Imaginary part */
chan->fftw_in[hrtf_length - i] = (chan->fftw_out[i] * chan->hrtf[hrtf_length - i]) +
(chan->fftw_out[hrtf_length - i] * chan->hrtf[i]);
}
/* The last (if even) FFTW result has never an imaginary part. */
if (hrtf_length % 2 == 0) {
chan->fftw_in[hrtf_length / 2] = chan->fftw_out[hrtf_length / 2] *
chan->hrtf[hrtf_length / 2];
}
/* iFFT */
fftw_execute(chan->fftw_plan_inverse);
/* Remove signal increase due to iFFT. */
for (i = 0; i < hrtf_length; i++) {
chan->fftw_out[i] = chan->fftw_out[i] / (hrtf_length / 2);
}
/* Save the block for overlapp add in the next itteration. */
for (i = 0; i < in_sample_size; i++) {
chan->overlap_add[i] += chan->fftw_out[i];
}
/* Copy real part to the output, ignore the complex part. */
for (i = 0; i < in_sample_size; i++) {
chan->out_data[i] = chan->overlap_add[i] * (SHRT_MAX / FLT_MAX);
chan->overlap_add[i] = chan->fftw_out[i + in_sample_size];
}
#endif
return 0;
}
struct convolve_channel_pair *do_convolve_pair(struct convolve_data *data,
unsigned int pos_id, int16_t *in_samples, unsigned int in_sample_size,
const char *channel_name)
{
struct convolve_channel_pair *chan_pair;
/* If a position has no active member we will not convolve. */
if (data->pos_ids[pos_id] != 1) {
ast_log(LOG_ERROR, "Channel %s: Channel pair has no active member! (pos id = %d)\n",
channel_name, pos_id);
return NULL;
}
chan_pair = data->cchan_pair[pos_id];
if (do_convolve(&chan_pair->chan_left, in_samples, in_sample_size, data->hrtf_length)) {
ast_log(LOG_ERROR, "Channel %s: Binaural processing failed.", channel_name);
return NULL;
}
if (do_convolve(&chan_pair->chan_right, in_samples, in_sample_size, data->hrtf_length)) {
ast_log(LOG_ERROR, "Channel %s: Binaural processing failed.", channel_name);
return NULL;
}
return chan_pair;
}
float *get_hrir(unsigned int chan_pos, unsigned int chan_side)
{
#ifdef BINAURAL_RENDERING
if (chan_side == HRIRS_CHANNEL_LEFT) {
return hrirs_left[ast_binaural_positions[chan_pos]];
} else if (chan_side == HRIRS_CHANNEL_RIGHT) {
return hrirs_right[ast_binaural_positions[chan_pos]];
}
#else
ast_log(LOG_ERROR, "Requesting data for the binaural conference feature without "
"it beeing active.\n");
#endif
return NULL;
}
int init_convolve_channel(struct convolve_channel *channel, unsigned int hrtf_len,
unsigned int chan_pos, unsigned int chan_side, unsigned int default_sample_size)
{
#ifdef BINAURAL_RENDERING
unsigned int j;
float *hrir;
/* Prepare FFTW. */
channel->fftw_in = fftw_alloc_real(hrtf_len + 1);
if (channel->fftw_in == NULL) {
return -1;
}
channel->fftw_out = fftw_alloc_real(hrtf_len + 1);
if (channel->fftw_out == NULL) {
fftw_free(channel->fftw_in);
return -1;
}
memset(channel->fftw_in, 0, sizeof(double) * (hrtf_len + 1));
memset(channel->fftw_out, 0, sizeof(double) * (hrtf_len + 1));
channel->fftw_plan = fftw_plan_r2r_1d(hrtf_len, channel->fftw_in, channel->fftw_out,
FFTW_R2HC, FFTW_PATIENT);
channel->fftw_plan_inverse = fftw_plan_r2r_1d(hrtf_len, channel->fftw_in, channel->fftw_out,
FFTW_HC2R, FFTW_PATIENT);
channel->out_data = ast_calloc(CONVOLVE_MAX_BUFFER, sizeof(int16_t));
if (channel->out_data == NULL) {
fftw_free(channel->fftw_in);
fftw_free(channel->fftw_out);
return -1;
}
/* Reuse positions if all positions are already used. */
chan_pos = chan_pos % HRIRS_IMPULSE_SIZE;
/* Get HRTF for the channels spatial position. */
hrir = get_hrir(chan_pos, chan_side);
if (hrir == NULL) {
fftw_free(channel->fftw_in);
fftw_free(channel->fftw_out);
ast_free(channel->out_data);
return -1;
}
for (j = 0; j < HRIRS_IMPULSE_LEN; j++) {
channel->fftw_in[j] = hrir[j];
}
for (j = HRIRS_IMPULSE_LEN; j < hrtf_len; j++) {
channel->fftw_in[j] = 0;
}
fftw_execute(channel->fftw_plan);
channel->hrtf = fftw_alloc_real(hrtf_len);
if (channel->hrtf == NULL) {
fftw_free(channel->fftw_in);
fftw_free(channel->fftw_out);
ast_free(channel->out_data);
return -1;
}
for (j = 0; j < hrtf_len; j++) {
channel->hrtf[j] = channel->fftw_out[j];
}
channel->overlap_add = ast_calloc(default_sample_size, sizeof(float));
return 0;
#endif
return -1;
}
int init_convolve_channel_pair(struct convolve_channel_pair *cchan_pair,
unsigned int hrtf_len, unsigned int chan_pos, unsigned int default_sample_size)
{
#ifdef BINAURAL_RENDERING
unsigned int hrirs_pos = chan_pos * 2;
int success = 0;
ast_debug(3, "Binaural pos for the new channel pair will be L: %d R: %d (pos id = %d)\n",
hrirs_pos, hrirs_pos + 1, chan_pos);
success = init_convolve_channel(&cchan_pair->chan_left, hrtf_len, chan_pos, HRIRS_CHANNEL_LEFT,
default_sample_size);
if (success == -1) {
return success;
}
success = init_convolve_channel(&cchan_pair->chan_right, hrtf_len, chan_pos,
HRIRS_CHANNEL_RIGHT, default_sample_size);
if (success == -1) {
free_convolve_channel(&cchan_pair->chan_left);
}
return success;
#else
ast_log(LOG_ERROR, "Requesting data for the binaural conference feature "
"without it beeing active.\n");
return -1;
#endif
}
int init_convolve_data(struct convolve_data *data, unsigned int default_sample_size)
{
unsigned int i;
unsigned int j;
int success;
success = 0;
data->pos_ids = ast_calloc(sizeof(int), sizeof(int) * CONVOLVE_CHANNEL_PREALLOC);
if (data->pos_ids == NULL) {
return -1;
}
data->chan_size = CONVOLVE_CHANNEL_PREALLOC;
data->number_channels = 0;
data->cchan_pair = ast_malloc(sizeof(struct convolve_channel_pair *) *
CONVOLVE_CHANNEL_PREALLOC);
if (data->cchan_pair == NULL) {
ast_free(data->pos_ids);
return -1;
}
for (i = 0; i < CONVOLVE_CHANNEL_PREALLOC; i++) {
data->cchan_pair[i] = ast_malloc(sizeof(struct convolve_channel_pair));
if (data->cchan_pair[i] == NULL) {
ast_free(data->pos_ids);
for (j = 0; j < i; j++) {
ast_free(data->cchan_pair[j]);
}
ast_free(data->cchan_pair);
return -1;
}
}
data->hrtf_length = (default_sample_size * 2) - 1;
for (i = 0; i < CONVOLVE_CHANNEL_PREALLOC; i++) {
success = init_convolve_channel_pair(data->cchan_pair[i], data->hrtf_length, i,
default_sample_size);
if (success == -1) {
ast_free(data->pos_ids);
for (j = 0; j < i; j++) {
free_convolve_channel_pair(data->cchan_pair[j]);
}
for (j = 0; j < CONVOLVE_CHANNEL_PREALLOC; j++) {
ast_free(data->cchan_pair[j]);
}
return -1;
}
}
return success;
}
void free_convolve_channel(struct convolve_channel *cchan)
{
#ifdef BINAURAL_RENDERING
fftw_free(cchan->fftw_out);
fftw_free(cchan->fftw_in);
fftw_free(cchan->hrtf);
ast_free(cchan->overlap_add);
ast_free(cchan->out_data);
fftw_destroy_plan(cchan->fftw_plan);
fftw_destroy_plan(cchan->fftw_plan_inverse);
#endif
}
void free_convolve_channel_pair(struct convolve_channel_pair *cchan_pair)
{
free_convolve_channel(&cchan_pair->chan_left);
free_convolve_channel(&cchan_pair->chan_right);
}
void free_convolve_data(struct convolve_data *data)
{
unsigned int i;
ast_free(data->pos_ids);
for (i = 0; i < data->chan_size; i++) {
free_convolve_channel_pair(data->cchan_pair[i]);
ast_free(data->cchan_pair[i]);
}
ast_free(data->cchan_pair);
}
int set_binaural_data_join(struct convolve_data *data, unsigned int default_sample_size)
{
struct convolve_channel_pair **cchan_pair_tmp;
unsigned int i;
int *pos_ids_tmp;
/* Raise the number of input channels. */
data->number_channels++;
/* We realloc another channel pair if we are out of prealloced ones. */
/* We have prealloced one at the beginning of a conference and if a member leaves. */
if (data->chan_size < data->number_channels) {
data->chan_size += 1;
pos_ids_tmp = ast_realloc(data->pos_ids, data->chan_size * sizeof(int));
if (pos_ids_tmp) {
data->pos_ids = pos_ids_tmp;
} else {
goto binaural_join_fails;
}
data->pos_ids[data->chan_size - 1] = 0;
cchan_pair_tmp = ast_realloc(data->cchan_pair,
data->chan_size * sizeof(struct convolve_channel_pair *));
if (cchan_pair_tmp) {
data->cchan_pair = cchan_pair_tmp;
} else {
goto binaural_join_fails;
}
data->cchan_pair[data->chan_size - 1] = ast_malloc(sizeof(struct convolve_channel_pair));
if (data->cchan_pair[data->chan_size - 1] == NULL) {
goto binaural_join_fails;
}
i = init_convolve_channel_pair(data->cchan_pair[data->chan_size - 1], data->hrtf_length,
data->chan_size - 1, default_sample_size);
if (i == -1) {
goto binaural_join_fails;
}
}
for (i = 0; i < data->chan_size; i++) {
if (data->pos_ids[i] == 0) {
data->pos_ids[i] = 1;
break;
}
}
return i;
binaural_join_fails:
data->number_channels--;
data->chan_size -= 1;
return -1;
}
void set_binaural_data_leave(struct convolve_data *data, unsigned int pos,
unsigned int default_sample_size)
{
if (pos >= data->chan_size || data->pos_ids[pos] == 0) {
return;
}
reset_channel_pair(data->cchan_pair[pos], default_sample_size);
data->number_channels--;
data->pos_ids[pos] = 0;
}
void softmix_process_write_binaural_audio(struct softmix_channel *sc,
unsigned int default_sample_size)
{
unsigned int i;
if (sc->write_frame.samples % default_sample_size != 0) {
return;
}
/* If binaural is suspended, the source audio (mono) will be removed. */
if (sc->binaural_suspended) {
for (i = 0; i < default_sample_size; i++) {
ast_slinear_saturated_subtract(&sc->final_buf[i * 2], &sc->our_buf[i]);
ast_slinear_saturated_subtract(&sc->final_buf[(i * 2) + 1], &sc->our_buf[i]);
}
return;
}
/* If binaural is NOT suspended, the source audio (binaural) will be removed. */
for (i = 0; i < default_sample_size; i++) {
ast_slinear_saturated_subtract(&sc->final_buf[i * 2],
&sc->our_chan_pair->chan_left.out_data[i]);
ast_slinear_saturated_subtract(&sc->final_buf[(i * 2) + 1],
&sc->our_chan_pair->chan_right.out_data[i]);
}
}
void check_binaural_position_change(struct ast_bridge *bridge,
struct softmix_bridge_data *softmix_data, struct ast_bridge_channel *bridge_channel)
{
unsigned int pos_change;
/*
* We only check binaural things if binaural is activated by the config
* and at least one binaural channel joined.
*/
if (!(bridge->softmix.binaural_active && softmix_data->convolve.binaural_active)) {
return;
}
/*
* Before we pull any audio, we must check if any channel requests a
* change of binaural positions.
*/
pos_change = 0;
AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
if (!bridge_channel->binaural_pos_change) {
continue;
}
ast_bridge_channel_lock_bridge(bridge_channel);
bridge_channel->binaural_pos_change = 0;
ast_bridge_unlock(bridge_channel->bridge);
pos_change = 1;
}
if (pos_change) {
random_binaural_pos_change(softmix_data);
}
}
void add_binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
unsigned int softmix_samples, struct softmix_mixing_array *mixing_array,
struct softmix_channel *sc, const char *channel_name)
{
struct convolve_channel_pair *pair;
pair = NULL;
/* We only check binaural things if at least one binaural channel joined. */
if (!(bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
&& (softmix_samples % CONVOLUTION_SAMPLE_SIZE) == 0)) {
return;
}
if (!sc->is_announcement) {
pair = do_convolve_pair(&softmix_data->convolve, sc->binaural_pos,
mixing_array->buffers[mixing_array->used_entries], softmix_samples, channel_name);
}
sc->our_chan_pair = pair;
mixing_array->chan_pairs[mixing_array->used_entries] = pair;
}
void binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
struct softmix_mixing_array *mixing_array, int16_t *bin_buf, int16_t *ann_buf)
{
unsigned int idx;
unsigned int x;
if (!(bridge->softmix.binaural_active && softmix_data->convolve.binaural_active)) {
return;
}
/* mix it like crazy (binaural channels) */
memset(bin_buf, 0, MAX_DATALEN);
memset(ann_buf, 0, MAX_DATALEN);
for (idx = 0; idx < mixing_array->used_entries; idx++) {
if (mixing_array->chan_pairs[idx] == NULL) {
for (x = 0; x < softmix_data->default_sample_size; x++) {
ast_slinear_saturated_add(bin_buf + (x * 2), mixing_array->buffers[idx] + x);
ast_slinear_saturated_add(bin_buf + (x * 2) + 1, mixing_array->buffers[idx] + x);
ann_buf[x * 2] = mixing_array->buffers[idx][x];
ann_buf[(x * 2) + 1] = mixing_array->buffers[idx][x];
}
} else {
for (x = 0; x < softmix_data->default_sample_size; x++) {
ast_slinear_saturated_add(bin_buf + (x * 2),
mixing_array->chan_pairs[idx]->chan_left.out_data + x);
ast_slinear_saturated_add(bin_buf + (x * 2) + 1,
mixing_array->chan_pairs[idx]->chan_right.out_data + x);
}
}
}
}
void create_binaural_frame(struct ast_bridge_channel *bridge_channel,
struct softmix_channel *sc, int16_t *bin_buf, int16_t *ann_buf,
unsigned int softmix_datalen, unsigned int softmix_samples, int16_t *buf)
{
unsigned int i;
sc->write_frame.datalen = softmix_datalen * 2;
sc->write_frame.samples = softmix_samples * 2;
if (!bridge_channel->binaural_suspended) {
sc->binaural_suspended = 0;
if (sc->is_announcement) {
memcpy(sc->final_buf, ann_buf, softmix_datalen * 2);
} else {
memcpy(sc->final_buf, bin_buf, softmix_datalen * 2);
}
return;
}
/*
* Mark that binaural output is suspended, since we use two channel audio
* we copy the same signals into both channels.
*/
sc->binaural_suspended = 1;
for (i = 0; i < softmix_samples; i++) {
sc->final_buf[i * 2] = buf[i];
sc->final_buf[(i * 2) + 1] = buf[i];
}
}

425
bridges/bridge_softmix/include/bridge_softmix_internal.h Normal file
View File

@@ -0,0 +1,425 @@
/*
* Asterisk -- An open source telephony toolkit.
*
* Copyright (C) 2011, Digium, Inc.
*
* Joshua Colp <jcolp@digium.com>
* David Vossel <dvossel@digium.com>
*
* See http://www.asterisk.org for more information about
* the Asterisk project. Please do not directly contact
* any of the maintainers of this project for assistance;
* the project provides a web site, mailing lists and IRC
* channels for your use.
*
* This program is free software, distributed under the terms of
* the GNU General Public License Version 2. See the LICENSE file
* at the top of the source tree.
*/
/*! \file
*
* \brief Multi-party software based channel mixing (header)
*
* \author Joshua Colp <jcolp@digium.com>
* \author David Vossel <dvossel@digium.com>
*
* \ingroup bridges
*/
#ifndef _ASTERISK_BRIDGE_SOFTMIX_INTERNAL_H
#define _ASTERISK_BRIDGE_SOFTMIX_INTERNAL_H
#include "asterisk.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <signal.h>
#include <errno.h>
#include <unistd.h>
#include "asterisk/module.h"
#include "asterisk/channel.h"
#include "asterisk/bridge.h"
#include "asterisk/bridge_technology.h"
#include "asterisk/frame.h"
#include "asterisk/options.h"
#include "asterisk/logger.h"
#include "asterisk/slinfactory.h"
#include "asterisk/astobj2.h"
#include "asterisk/timing.h"
#include "asterisk/translate.h"
#ifdef BINAURAL_RENDERING
#include <fftw3.h>
#endif
#if defined(__Darwin__) || defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__CYGWIN__)
#include <float.h>
#else
#include <values.h>
#endif
#define MAX_DATALEN 8096
#define DEFAULT_ENERGY_HISTORY_LEN 150
/*! Setting the sample rate to 48000 by default if binaural is activated. */
#define SOFTMIX_BINAURAL_SAMPLE_RATE 48000
/*! We only support 20 ms interval length with binaural data at the moment. */
#define BINAURAL_MIXING_INTERVAL 20
struct convolve_channel {
/*! The head related transfer function used for convolving */
double *hrtf;
/*! Input signals for fftw */
double *fftw_in;
/*! Output signals from the fftw */
double *fftw_out;
/*! Signals for overlap add */
float *overlap_add;
/*! The resulting data after the convolution */
int16_t *out_data;
#ifdef BINAURAL_RENDERING
/*! The fftw plan for binaural signaling */
fftw_plan fftw_plan;
/*! The inverse fftw plan for binaural signaling */
fftw_plan fftw_plan_inverse;
#endif
};
struct convolve_channel_pair {
/*! The left channel of a stereo channel pair */
struct convolve_channel chan_left;
/*! The right channel of a stereo channel pair */
struct convolve_channel chan_right;
};
struct convolve_data {
/*! A count of all channels potentialy having input data for the conference. */
int number_channels;
/*! Will set to true if there is at least one binaural output.
* Only if set to true data will be convolved. */
int binaural_active;
/*! The length of the head related transfer function */
unsigned int hrtf_length;
/*! Number of channels available for convolving.
* We do not delete a channel when a member leaves, cause we can reuse it for the next one. */
int chan_size;
/*! The positions of the single channels in the virtual room */
int *pos_ids;
/*! Each channel has a stereo pair of channels for the convolution */
struct convolve_channel_pair **cchan_pair;
};
struct video_follow_talker_data {
/*! audio energy history */
int energy_history[DEFAULT_ENERGY_HISTORY_LEN];
/*! The current slot being used in the history buffer, this
* increments and wraps around */
int energy_history_cur_slot;
/*! The current energy sum used for averages. */
int energy_accum;
/*! The current energy average */
int energy_average;
};
/*! \brief Structure which contains per-channel mixing information */
struct softmix_channel {
/*! Lock to protect this structure */
ast_mutex_t lock;
/*! Factory which contains audio read in from the channel */
struct ast_slinfactory factory;
/*! Frame that contains mixed audio to be written out to the channel */
struct ast_frame write_frame;
/*! Current expected read slinear format. */
struct ast_format *read_slin_format;
/*! DSP for detecting silence */
struct ast_dsp *dsp;
/*!
* \brief TRUE if a channel is talking.
*
* \note This affects how the channel's audio is mixed back to
* it.
*/
unsigned int talking:1;
/*! TRUE if the channel provided audio for this mixing interval */
unsigned int have_audio:1;
/*! We set binaural also as channel data, to have better tracking.
* It is also present in transpvt.
*/
unsigned int binaural:1;
/*! TRUE if this is an announcement channel (data will not be convolved) */
unsigned int is_announcement:1;
/*! The position of the channel in the virtual room represented by an id
* This ID has to be set even if the channel has no binaural output!
*/
unsigned int binaural_pos;
/*! The channel pair for this channel */
struct convolve_channel_pair *our_chan_pair;
/*! Marks the channel for suspending all binaural activity on the output */
unsigned int binaural_suspended:1;
/*! Channel sample rate, stored to retrieve it after unsuspending the channel */
int rate;
/*! Buffer containing final mixed audio from all sources */
short final_buf[MAX_DATALEN];
/*! Buffer containing only the audio from the channel */
short our_buf[MAX_DATALEN];
/*! Data pertaining to talker mode for video conferencing */
struct video_follow_talker_data video_talker;
};
struct softmix_bridge_data {
struct ast_timer *timer;
/*!
* \brief Bridge pointer passed to the softmix mixing thread.
*
* \note Does not need a reference because the bridge will
* always exist while the mixing thread exists even if the
* bridge is no longer actively using the softmix technology.
*/
struct ast_bridge *bridge;
/*! Lock for signaling the mixing thread. */
ast_mutex_t lock;
/*! Condition, used if we need to wake up the mixing thread. */
ast_cond_t cond;
/*! Thread handling the mixing */
pthread_t thread;
unsigned int internal_rate;
unsigned int internal_mixing_interval;
/*! TRUE if the mixing thread should stop */
unsigned int stop:1;
/*! The default sample size (e.g. using Opus at 48khz and 20 ms mixing
* interval, sample size is 960) */
unsigned int default_sample_size;
/*! All data needed for binaural signaling */
struct convolve_data convolve;
/*! TRUE if the first attempt to init binaural rendering data was done
* (does not guarantee success)
*/
unsigned int binaural_init;
};
struct softmix_mixing_array {
unsigned int max_num_entries;
unsigned int used_entries;
int16_t **buffers;
/*! Stereo channel pairs used to store convolved binaural signals */
struct convolve_channel_pair **chan_pairs;
};
/*!
* \brief Deletes left over signals on a channel that it can be reused.
*
* \param channel_pair The channel pair which contains the left and right audio channel.
* \param default_sample_size The sample size which the channel pair uses.
*/
void reset_channel_pair(struct convolve_channel_pair *channel_pair,
unsigned int default_sample_size);
/*!
* \brief Randomly changes the virtual positions of conference participants.
*
* \param softmix_data The structure containing all position informations.
*/
void random_binaural_pos_change(struct softmix_bridge_data *softmix_data);
/*!
* \brief Binaural convolving of audio data for a channel.
*
* \param chan The channel that will contain the binaural audio data as result.
* \param in_samples The audio data which will be convolved.
* \param in_sample_size The size of the audio data.
* \param hrtf_length The length of the head related transfer function used to convolve the audio.
*
* \retval 0 success
* \retval -1 failure
*/
int do_convolve(struct convolve_channel *chan, int16_t *in_samples,
unsigned int in_sample_size, unsigned int hrtf_length);
/*!
* \brief Binaural convolving of audio data for a channel pair (left and right channel).
*
* \param data Contains the left and right audio channel.
* \param pos_id The position the channel has in the virtual enviroment.
* \param in_samples The audio data which will be convolved for both channels.
* \param in_sample_size The size of the audio data.
* \param channel_name The name of the channel
*
* \retval The channel pair with convolved audio on success.
* \retval NULL failure
*/
struct convolve_channel_pair *do_convolve_pair(struct convolve_data *data,
unsigned int pos_id, int16_t *in_samples, unsigned int in_sample_size,
const char *channel_name);
/*!
* \brief Provides a head related impulse response for the given position in the virtual
* enviroment.
*
* \param chan_pos The position of the channel in the virtual enviroment.
* \param chan_side 0 for the left audio channel, 1 for the right.
*
* \retval The hrir for the given position in the virtual room for either the left or right
* channels.
* \retval NULL on failure.
*
*/
float *get_hrir(unsigned int chan_pos, unsigned int chan_side);
/*!
* \brief Initializes all data needed for binaural audio processing.
*
* \param channel The channel used for binaural audio processing.
* \param hrtf_len The length of the head related impulse response used for binaural processing.
* \param chan_pos The position of the channel in the virtual enviroment.
* \param chan_side 0 for the left audio channel, 1 for the right.
* \param default_sample_size The default size of audio samples.
*
* \retval 0 on success
* \retval -1 on failure
*/
int init_convolve_channel(struct convolve_channel *channel, unsigned int hrtf_len,
unsigned int chan_pos, unsigned int chan_side, unsigned int default_sample_size);
/*!
* \brief Initializies all data needed for binaural audio processing of a channel pair
* (left and right).
*
* \param cchan_pair The channel pair used for binaural audio processing.
* \param hrtf_len The length of the head related impulse response used for binaural processing.
* \param chan_pos The position of the channel in the virtual enviroment.
* \param default_sample_size The default size of audio samples.
*
* \retval 0 on success
* \retval -1 on failure
*/
int init_convolve_channel_pair(struct convolve_channel_pair *cchan_pair,
unsigned int hrtf_len, unsigned int chan_pos, unsigned int default_sample_size);
/*!
* \brief Preinits a specific number of channels (CONVOVLE_CHANNEL_PREALLOC)
* at the beginning of a conference.
*
* \param data Contains all channels and data needed for binaural processing
* (e.g. head related transfer functions).
* \param default_sample_size The default size of audio samples.
*
* \retval 0 on success
* \retval -1 on failure
*/
int init_convolve_data(struct convolve_data *data, unsigned int default_sample_size);
/*!
* \brief Frees all data needed for binaural processing by an audio channel.
*
* \param cchan The channel to clean up.
*/
void free_convolve_channel(struct convolve_channel *cchan);
/*!
* \brief Frees all data needed for binaural processing by a pair of audio channels
* (left and right).
*
* \param cchan_pair The channel pair to clean up.
*/
void free_convolve_channel_pair(struct convolve_channel_pair *cchan_pair);
/*!
* \brief Frees all channels and data needed for binaural audio processing.
*
* \param data Contains all channels and data for the cleanup process.
*/
void free_convolve_data(struct convolve_data *data);
/*!
* \brief Joins a channel into a virtual enviroment build with the help of binaural sythesis.
*
* \param data Contains all channels and data needed for binaural processing
* (e.g. head related transfer functions).
* \param default_sample_size The default size of audio samples.
*
* \retval The position of the channel in the virtual enviroment.
* \retval -1 on failure
*/
int set_binaural_data_join(struct convolve_data *data, unsigned int default_sample_size);
/*!
* \brief Removes a channel from the binaural conference bridge. Marks the position in
* the virtual room as unused that it can be reused by the next channel which enters the
* conference.
*
* \param data Contains all channels and data needed for binaural processing
* (e.g. head related transfer functions).
* \param pos The position of the channel in the virtual enviroment.
* \param default_sample_size The default size of audio samples.
*/
void set_binaural_data_leave(struct convolve_data *data, unsigned int pos,
unsigned int default_sample_size);
/*!
* \brief Writes the binaural audio to a channel.
*
* \param sc The softmix channel.
* \param default_sample_size The default size of audio samples.
*/
void softmix_process_write_binaural_audio(struct softmix_channel *sc,
unsigned int default_sample_size);
/*!
* \brief Checks if a position change in the virual enviroment is requested by one of
* the participants.
*
* \param bridge The conference bridge.
* \param softmix_data The data used by the softmix bridge.
* \param bridge_channel The bridge channel.
*/
void check_binaural_position_change(struct ast_bridge *bridge,
struct softmix_bridge_data *softmix_data, struct ast_bridge_channel *bridge_channel);
/*!
* \brief Processes audio data with the binaural synthesis and adds the result to the mixing array.
*
* \param bridge The conference bridge needed to check if binaural processing is active or not.
* \param softmix_data Contains all data for the softmix bridge and for the binaural processing.
* \param softmix_samples The sample size.
* \param mixing_array The array which holds all audio data for mixing.
* \param sc The channel which contains the audio data to process.
* \param channel_name The name of the channel
*/
void add_binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
unsigned int softmix_samples, struct softmix_mixing_array *mixing_array,
struct softmix_channel *sc, const char *channel_name);
/*!
* \brief Mixes all binaural audio data contained in the mixing array.
*
* \param bridge The conference bridge needed to check if binaural processing is active or not.
* \param softmix_data Contains all data for the softmix bridge and for the binaural processing.
* \param mixing_array The array which holds all audio data for mixing.
* \param bin_buf The buffer that will contain the mixing results.
* \param ann_buf The buffer that will contain mixed announcements in an interleaved format.
*/
void binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
struct softmix_mixing_array *mixing_array, int16_t *bin_buf, int16_t *ann_buf);
/*!
* \brief Creates a frame out of binaural audio data.
*
* \param bridge_channel Contains the information if binaural processing is active or not.
* If active binaural audio data will be copied, if not mono data will be provided in an
* interleaved format.
* \param sc The softmix channel holding all informations for the process.
* \param bin_buf The buffer that contains all mixing results.
* \param ann_buf The buffer that contains mixed announcements in an interleaved format.
* \param softmix_datalen The size of the audio data.
* \param softmix_samples The number of audio samples.
* \param buf The buffer that contains all mono mixing results, used if binaural processing is
* inactive.
*/
void create_binaural_frame(struct ast_bridge_channel *bridge_channel,
struct softmix_channel *sc, int16_t *bin_buf, int16_t *ann_buf,
unsigned int softmix_datalen, unsigned int softmix_samples, int16_t *buf);
#endif /* _ASTERISK_BRIDGE_SOFTMIX_INTERNAL_H */

50730
bridges/bridge_softmix/include/hrirs.h Normal file
View File

File diff suppressed because it is too large Load Diff

63
bridges/bridge_softmix/include/hrirs_configuration.h Normal file
View File

@@ -0,0 +1,63 @@
/*
* Asterisk -- An open source telephony toolkit.
*
* Copyright (C) 2016, Frank Haase, Dennis Guse
*
* Frank Haase <fra.haase@gmail.com>
* Dennis Guse <dennis.guse@alumni.tu-berlin.de>
*
* See http://www.asterisk.org for more information about
* the Asterisk project. Please do not directly contact
* any of the maintainers of this project for assistance;
* the project provides a web site, mailing lists and IRC
* channels for your use.
*
* This program is free software, distributed under the terms of
* the GNU General Public License Version 2. See the LICENSE file
* at the top of the source tree.
*/
/*! \file
*
* \brief Multi-party software binaural channel mixing (header)
*
* \author Frank Haase <fra.haase@googlemail.com>
* \author Dennis Guse <dennis.guse@alumni.tu-berlin.de>
*
* \ingroup bridges
*/
#ifndef _ASTERISK_HRIRS_CONFIGURATION_H
#define _ASTERISK_HRIRS_CONFIGURATION_H
#include "hrirs.h"
/*! The size of possible positions in the virtual enviroment build with the help
* of binaural audio processing.
*/
#define POSITION_SIZE 181
#if POSITION_SIZE != HRIRS_IMPULSE_SIZE
#error "The conference is designed for 181 individual places at the moment. If you want to change this please alter the positions array first."
#endif
/*! The offset for the left channel audio channel. */
#define HRIRS_CHANNEL_LEFT 0
/*! The offset for the right channel audio channel. */
#define HRIRS_CHANNEL_RIGHT 1
/*! The ast_binaural_positions array contains a specific plan to order conference
* participants in the virtual enviroment.
*/
static unsigned int ast_binaural_positions[POSITION_SIZE] = {
90, 80, 100, 70, 110, 60, 120, 50, 130, 40, 140, 20, 160, 0, 180, 85, 95, 75, 105, 65, 115,
55, 125, 45, 135, 30, 150, 10, 170, 87, 93, 82, 98, 77, 103, 72, 108, 67, 113, 62, 118, 57,
123, 52, 128, 47, 133, 42, 138, 35, 145, 25, 155, 15, 165, 5, 175, 88, 92, 86, 83, 97, 81,
99, 78, 102, 76, 104, 73, 94, 107, 71, 109, 68, 112, 66, 114, 63, 117, 61, 119, 58, 122, 56,
124, 53, 127, 51, 129, 48, 132, 46, 134, 43, 137, 41, 139, 37, 143, 32, 148, 27, 153, 22,
158, 17, 163, 12, 168, 7, 96, 173, 2, 178, 89, 91, 84, 79, 101, 74, 106, 69, 111, 64, 116,
59, 121, 54, 126, 49, 131, 44, 136, 38, 142, 36, 144, 33, 147, 31, 149, 28, 152, 26, 154,
23, 157, 21, 159, 162, 16, 164, 13, 167, 11, 169, 8, 172, 6, 174, 3, 177, 1, 179, 39, 141,
34, 146, 29, 151, 24, 156, 19, 161, 14, 166, 9, 171, 4, 18, 176 };
#endif /* _ASTERISK_HRIRS_CONFIGURATION_H */