Instructions to use HKUSTAudio/VidMuse with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Audiocraft
How to use HKUSTAudio/VidMuse with Audiocraft:
# Type of model unknown.
- Notebooks
- Google Colab
- Kaggle
Download audiocraft/data/video.py from HKUSTAudio/VidMuse: direct link, hf CLI and curl.
- Browser
- Download file 3.78 kB
-
https://huggingface.co/HKUSTAudio/VidMuse/resolve/main/audiocraft/data/video.py
- Command line
-
hf download hf://HKUSTAudio/VidMuse/audiocraft/data/video.py
-
curl -L -o video.py https://huggingface.co/HKUSTAudio/VidMuse/resolve/main/audiocraft/data/video.py
3.78 kB
| import decord | |
| from decord import VideoReader | |
| from decord import cpu | |
| import torch | |
| import math | |
| import einops | |
| import torchvision.transforms as transforms | |
| def adjust_video_duration(video_tensor, duration, target_fps): | |
| current_duration = video_tensor.shape[1] | |
| target_duration = duration * target_fps | |
| if current_duration > target_duration: | |
| video_tensor = video_tensor[:, :target_duration] | |
| elif current_duration < target_duration: | |
| last_frame = video_tensor[:, -1:] | |
| repeat_times = target_duration - current_duration | |
| video_tensor = torch.cat((video_tensor, last_frame.repeat(1, repeat_times, 1, 1)), dim=1) | |
| return video_tensor | |
| def video_read_local(filepath, seek_time=0., duration=-1, target_fps=2): | |
| vr = VideoReader(filepath, ctx=cpu(0)) | |
| fps = vr.get_avg_fps() | |
| if duration > 0: | |
| total_frames_to_read = target_fps * duration | |
| frame_interval = int(math.ceil(fps / target_fps)) | |
| start_frame = int(seek_time * fps) | |
| end_frame = start_frame + frame_interval * total_frames_to_read | |
| frame_ids = list(range(start_frame, min(end_frame, len(vr)), frame_interval)) | |
| else: | |
| frame_ids = list(range(0, len(vr), int(math.ceil(fps / target_fps)))) | |
| frames = vr.get_batch(frame_ids) | |
| frames = torch.from_numpy(frames.asnumpy()).permute(0, 3, 1, 2) # [N, H, W, C] -> [N, C, H, W] | |
| resize_transform = transforms.Resize((224, 224)) | |
| frames = [resize_transform(frame) for frame in frames] | |
| video_tensor = torch.stack(frames) | |
| video_tensor = einops.rearrange(video_tensor, 't c h w -> c t h w') # [T, C, H, W] -> [C, T, H, W] | |
| video_tensor = adjust_video_duration(video_tensor, duration, target_fps) | |
| assert video_tensor.shape[1] == duration * target_fps, f"the shape of video_tensor is {video_tensor.shape}" | |
| return video_tensor | |
| def video_read_global(filepath, seek_time=0., duration=-1, target_fps=2, global_mode='average', global_num_frames=32): | |
| vr = VideoReader(filepath, ctx=cpu(0)) | |
| fps = vr.get_avg_fps() | |
| frame_count = len(vr) | |
| if duration > 0: | |
| total_frames_to_read = target_fps * duration | |
| frame_interval = int(math.ceil(fps / target_fps)) | |
| start_frame = int(seek_time * fps) | |
| end_frame = start_frame + frame_interval * total_frames_to_read | |
| frame_ids = list(range(start_frame, min(end_frame, frame_count), frame_interval)) | |
| else: | |
| frame_ids = list(range(0, frame_count, int(math.ceil(fps / target_fps)))) | |
| local_frames = vr.get_batch(frame_ids) | |
| local_frames = torch.from_numpy(local_frames.asnumpy()).permute(0, 3, 1, 2) # [N, H, W, C] -> [N, C, H, W] | |
| resize_transform = transforms.Resize((224, 224)) | |
| local_frames = [resize_transform(frame) for frame in local_frames] | |
| local_video_tensor = torch.stack(local_frames) | |
| local_video_tensor = einops.rearrange(local_video_tensor, 't c h w -> c t h w') # [T, C, H, W] -> [C, T, H, W] | |
| local_video_tensor = adjust_video_duration(local_video_tensor, duration, target_fps) | |
| if global_mode=='average': | |
| global_frame_ids = torch.linspace(0, frame_count - 1, global_num_frames).long() | |
| global_frames = vr.get_batch(global_frame_ids) | |
| global_frames = torch.from_numpy(global_frames.asnumpy()).permute(0, 3, 1, 2) # [N, H, W, C] -> [N, C, H, W] | |
| global_frames = [resize_transform(frame) for frame in global_frames] | |
| global_video_tensor = torch.stack(global_frames) | |
| global_video_tensor = einops.rearrange(global_video_tensor, 't c h w -> c t h w') # [T, C, H, W] -> [C, T, H, W] | |
| assert global_video_tensor.shape[1] == global_num_frames, f"the shape of global_video_tensor is {global_video_tensor.shape}" | |
| return local_video_tensor, global_video_tensor | |