Add Thumos feature extraction

48cc3c3f · Arian Bakhtiarnia · ff884e6e · 48cc3c3f · 48cc3c3f
Commit 48cc3c3f authored Jan 14, 2022 by Arian Bakhtiarnia
--- a/README.md
+++ b/README.md
@@ -11,4 +11,4 @@ This code was developed and executed using Jupyter notebooks. The following inst
 5. <kbd>CTRL</kbd>+<kbd>P</kbd> then <kbd>CTRL</kbd>+<kbd>Q</kbd> to detach from the container without terminating the execution.
 6. Paste the copied URL in your browser to open Jupyter (if you are running the docker container on a remote server, you need to replace the IP address with that of the server).
 7. `git clone https://github.com/LukasHedegaard/continual-transformers`
-8. Move `continual_transformer_audio_classification.ipynb` inside the `notebooks` directory of the above repository and run it.
+8. Move `.ipynb` files inside the `notebooks` directory of the above repository and run them.
--- a/extract_thumos_audio_features.ipynb
+++ b/extract_thumos_audio_features.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SELECTED_GPUS = [7]\n",
+    "\n",
+    "import os\n",
+    "\n",
+    "os.environ['CUDA_VISIBLE_DEVICES'] = ','.join([str(gpu_number) for gpu_number in SELECTED_GPUS])\n",
+    "\n",
+    "import tensorflow as tf \n",
+    "\n",
+    "tf.get_logger().setLevel('INFO')\n",
+    "\n",
+    "assert len(tf.config.list_physical_devices('GPU')) > 0\n",
+    "\n",
+    "GPUS = tf.config.experimental.list_physical_devices('GPU')\n",
+    "for gpu in GPUS:\n",
+    "    tf.config.experimental.set_memory_growth(gpu, True)\n",
+    "\n",
+    "ROOT_DIR = '..'\n",
+    "os.chdir(ROOT_DIR)\n",
+    "\n",
+    "import librosa\n",
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "import pickle\n",
+    "import sys\n",
+    "import tensorflow as tf\n",
+    "import wave\n",
+    "\n",
+    "from preprocess_sound import preprocess_sound\n",
+    "from subprocess import check_output\n",
+    "\n",
+    "VIDEO_DIR = os.path.join('thumos14', 'data')\n",
+    "FEATURES_DIR = os.path.join('thumos14', 'featuresv2')\n",
+    "def ensure_dir(directory):\n",
+    "    if not os.path.exists(directory):\n",
+    "        os.makedirs(directory)\n",
+    "ensure_dir(FEATURES_DIR)\n",
+    "TMP_WAV_PATH = 'tmp.wav'\n",
+    "FPS = 5\n",
+    "AUDIO_SIZE = (96, 64, 1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def VGGish(load_weights=True, weights='audioset',\n",
+    "           input_tensor=None, input_shape=AUDIO_SIZE,\n",
+    "           out_dim=128, include_top=True, pooling='avg'):\n",
+    "    if weights not in {'audioset', None}:\n",
+    "        raise ValueError('The `weights` argument should be either '\n",
+    "                         '`None` (random initialization) or `audioset` '\n",
+    "                         '(pre-training on audioset).')\n",
+    "    if input_tensor is None:\n",
+    "        aud_input = tf.keras.layers.Input(shape=input_shape, name='input_1')\n",
+    "    else:\n",
+    "        aud_input = input_tensor\n",
+    "\n",
+    "    x = tf.keras.layers.Conv2D(64, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv1')(aud_input)\n",
+    "    x = tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool1')(x)\n",
+    "\n",
+    "    x = tf.keras.layers.Conv2D(128, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv2')(x)\n",
+    "    x = tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool2')(x)\n",
+    "\n",
+    "    x = tf.keras.layers.Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv3/conv3_1')(x)\n",
+    "    x = tf.keras.layers.Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv3/conv3_2')(x)\n",
+    "    x = tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool3')(x)\n",
+    "\n",
+    "    x = tf.keras.layers.Conv2D(512, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv4/conv4_1')(x)\n",
+    "    x = tf.keras.layers.Conv2D(512, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv4/conv4_2')(x)\n",
+    "    x = tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool4')(x)\n",
+    "\n",
+    "    if include_top:\n",
+    "        x = tf.keras.layers.Flatten(name='flatten_')(x)\n",
+    "        x = tf.keras.layers.Dense(4096, activation='relu', name='vggish_fc1/fc1_1')(x)\n",
+    "        x = tf.keras.layers.Dense(4096, activation='relu', name='vggish_fc1/fc1_2')(x)\n",
+    "        x = tf.keras.layers.Dense(out_dim, activation='relu', name='vggish_fc2')(x)\n",
+    "    else:\n",
+    "        if pooling == 'avg':\n",
+    "            x = tf.keras.layers.GlobalAveragePooling2D()(x)\n",
+    "        elif pooling == 'max':\n",
+    "            x = tf.keras.layers.GlobalMaxPooling2D()(x)\n",
+    "\n",
+    "    model = tf.keras.models.Model(aud_input, x, name='VGGish')\n",
+    "\n",
+    "    if load_weights:\n",
+    "        if weights == 'audioset':\n",
+    "            if include_top:\n",
+    "                model.load_weights('vggish_audioset_weights.h5')\n",
+    "            else:\n",
+    "                model.load_weights('vggish_audioset_weights_without_fc2.h5')\n",
+    "        else:\n",
+    "            print(\"failed to load weights\")\n",
+    "\n",
+    "    return model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def extract_features():\n",
+    "    backbone_model = VGGish(\n",
+    "        include_top=True,\n",
+    "        load_weights=True,\n",
+    "        input_shape=AUDIO_SIZE\n",
+    "    )\n",
+    "    model = tf.keras.models.Model(\n",
+    "        inputs=backbone_model.get_layer(index=0).input,\n",
+    "        outputs=backbone_model.get_layer(index=-2).output\n",
+    "    )\n",
+    "    for sub_dir in ['test', 'val']:\n",
+    "        print(sub_dir)\n",
+    "        video_sub_dir = os.path.join(VIDEO_DIR, sub_dir)\n",
+    "        total_files = len(\n",
+    "            [name for name in os.listdir(video_sub_dir) if os.path.isfile(os.path.join(video_sub_dir, name))]\n",
+    "        )\n",
+    "        features_sub_dir = os.path.join(FEATURES_DIR, sub_dir)\n",
+    "        ensure_dir(features_sub_dir)\n",
+    "        for i, file_name in enumerate(os.listdir(video_sub_dir)):\n",
+    "            sys.stdout.write('\\r[%d/%d]' % (i + 1, total_files))\n",
+    "            sys.stdout.flush()\n",
+    "            if file_name.endswith('.mp4'):\n",
+    "                feature_file_path = os.path.join(features_sub_dir, file_name.replace('.mp4', '.pkl'))\n",
+    "                if not os.path.exists(feature_file_path):\n",
+    "                    check_output([\n",
+    "                        'ffmpeg',\n",
+    "                        '-y',\n",
+    "                        '-i',\n",
+    "                        os.path.join(video_sub_dir, file_name),\n",
+    "                        TMP_WAV_PATH,\n",
+    "                    ])\n",
+    "                    with wave.open(TMP_WAV_PATH, 'rb') as wave_file:\n",
+    "                        sampling_rate = wave_file.getframerate()\n",
+    "                    waveform, _ = librosa.load(TMP_WAV_PATH, sr=sampling_rate)\n",
+    "                    features = []\n",
+    "                    for start_index in range(0, waveform.shape[0], sampling_rate // FPS):\n",
+    "                        sub_waveform = waveform[start_index:start_index + sampling_rate]\n",
+    "                        if sub_waveform.shape[0] < sampling_rate:\n",
+    "                            zero_padding = np.zeros(sampling_rate - sub_waveform.shape[0])\n",
+    "                            sub_waveform = np.concatenate([sub_waveform, zero_padding])\n",
+    "                        spectrogram = preprocess_sound(sub_waveform, sampling_rate)\n",
+    "                        spectrogram = np.moveaxis(spectrogram, 0, -1)\n",
+    "                        output = model(np.expand_dims(spectrogram, axis=0))\n",
+    "                        features.append(output[0])\n",
+    "                    features = np.array(features)\n",
+    "                    with open(feature_file_path, 'wb') as feature_file:\n",
+    "                        pickle.dump(features, feature_file)\n",
+    "        print()  # newline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "extract_features()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
+%% Cell type:code id: tags:
+
+``` python
+SELECTED_GPUS = [7]
+
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = ','.join([str(gpu_number) for gpu_number in SELECTED_GPUS])
+
+import tensorflow as tf
+
+tf.get_logger().setLevel('INFO')
+
+assert len(tf.config.list_physical_devices('GPU')) > 0
+
+GPUS = tf.config.experimental.list_physical_devices('GPU')
+for gpu in GPUS:
+    tf.config.experimental.set_memory_growth(gpu, True)
+
+ROOT_DIR = '..'
+os.chdir(ROOT_DIR)
+
+import librosa
+import matplotlib.pyplot as plt
+import numpy as np
+import pickle
+import sys
+import tensorflow as tf
+import wave
+
+from preprocess_sound import preprocess_sound
+from subprocess import check_output
+
+VIDEO_DIR = os.path.join('thumos14', 'data')
+FEATURES_DIR = os.path.join('thumos14', 'featuresv2')
+def ensure_dir(directory):
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+ensure_dir(FEATURES_DIR)
+TMP_WAV_PATH = 'tmp.wav'
+FPS = 5
+AUDIO_SIZE = (96, 64, 1)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+def VGGish(load_weights=True, weights='audioset',
+           input_tensor=None, input_shape=AUDIO_SIZE,
+           out_dim=128, include_top=True, pooling='avg'):
+    if weights not in {'audioset', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `audioset` '
+                         '(pre-training on audioset).')
+    if input_tensor is None:
+        aud_input = tf.keras.layers.Input(shape=input_shape, name='input_1')
+    else:
+        aud_input = input_tensor
+
+    x = tf.keras.layers.Conv2D(64, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv1')(aud_input)
+    x = tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool1')(x)
+
+    x = tf.keras.layers.Conv2D(128, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv2')(x)
+    x = tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool2')(x)
+
+    x = tf.keras.layers.Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv3/conv3_1')(x)
+    x = tf.keras.layers.Conv2D(256, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv3/conv3_2')(x)
+    x = tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool3')(x)
+
+    x = tf.keras.layers.Conv2D(512, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv4/conv4_1')(x)
+    x = tf.keras.layers.Conv2D(512, (3, 3), strides=(1, 1), activation='relu', padding='same', name='conv4/conv4_2')(x)
+    x = tf.keras.layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='pool4')(x)
+
+    if include_top:
+        x = tf.keras.layers.Flatten(name='flatten_')(x)
+        x = tf.keras.layers.Dense(4096, activation='relu', name='vggish_fc1/fc1_1')(x)
+        x = tf.keras.layers.Dense(4096, activation='relu', name='vggish_fc1/fc1_2')(x)
+        x = tf.keras.layers.Dense(out_dim, activation='relu', name='vggish_fc2')(x)
+    else:
+        if pooling == 'avg':
+            x = tf.keras.layers.GlobalAveragePooling2D()(x)
+        elif pooling == 'max':
+            x = tf.keras.layers.GlobalMaxPooling2D()(x)
+
+    model = tf.keras.models.Model(aud_input, x, name='VGGish')
+
+    if load_weights:
+        if weights == 'audioset':
+            if include_top:
+                model.load_weights('vggish_audioset_weights.h5')
+            else:
+                model.load_weights('vggish_audioset_weights_without_fc2.h5')
+        else:
+            print("failed to load weights")
+
+    return model
+```
+
+%% Cell type:code id: tags:
+
+``` python
+def extract_features():
+    backbone_model = VGGish(
+        include_top=True,
+        load_weights=True,
+        input_shape=AUDIO_SIZE
+    )
+    model = tf.keras.models.Model(
+        inputs=backbone_model.get_layer(index=0).input,
+        outputs=backbone_model.get_layer(index=-2).output
+    )
+    for sub_dir in ['test', 'val']:
+        print(sub_dir)
+        video_sub_dir = os.path.join(VIDEO_DIR, sub_dir)
+        total_files = len(
+            [name for name in os.listdir(video_sub_dir) if os.path.isfile(os.path.join(video_sub_dir, name))]
+        )
+        features_sub_dir = os.path.join(FEATURES_DIR, sub_dir)
+        ensure_dir(features_sub_dir)
+        for i, file_name in enumerate(os.listdir(video_sub_dir)):
+            sys.stdout.write('\r[%d/%d]' % (i + 1, total_files))
+            sys.stdout.flush()
+            if file_name.endswith('.mp4'):
+                feature_file_path = os.path.join(features_sub_dir, file_name.replace('.mp4', '.pkl'))
+                if not os.path.exists(feature_file_path):
+                    check_output([
+                        'ffmpeg',
+                        '-y',
+                        '-i',
+                        os.path.join(video_sub_dir, file_name),
+                        TMP_WAV_PATH,
+                    ])
+                    with wave.open(TMP_WAV_PATH, 'rb') as wave_file:
+                        sampling_rate = wave_file.getframerate()
+                    waveform, _ = librosa.load(TMP_WAV_PATH, sr=sampling_rate)
+                    features = []
+                    for start_index in range(0, waveform.shape[0], sampling_rate // FPS):
+                        sub_waveform = waveform[start_index:start_index + sampling_rate]
+                        if sub_waveform.shape[0] < sampling_rate:
+                            zero_padding = np.zeros(sampling_rate - sub_waveform.shape[0])
+                            sub_waveform = np.concatenate([sub_waveform, zero_padding])
+                        spectrogram = preprocess_sound(sub_waveform, sampling_rate)
+                        spectrogram = np.moveaxis(spectrogram, 0, -1)
+                        output = model(np.expand_dims(spectrogram, axis=0))
+                        features.append(output[0])
+                    features = np.array(features)
+                    with open(feature_file_path, 'wb') as feature_file:
+                        pickle.dump(features, feature_file)
+        print()  # newline
+```
+
+%% Cell type:code id: tags:
+
+``` python
+extract_features()
+```