如何使用 Tensorboard Embedding Projector 将 Pytorch 与自定义数据集和自定义模型结合使用

问：

目前我正在做图像嵌入可视化，我想使用 Tensorboard 投影仪 PCA 和 T-SNE 来查看图像嵌入的相似性。我按照网站代码进行可视化，但我无法进行预期的可视化，其中相同的图像应该聚集在一起，但它只是显示不同的图像聚集在一起。我不使用标签或元数据，因为我在做无监督学习。对于 PCA，我将随机图像围成一个圆圈，但对于 T-SNE，我什么也没显示，只有 0 或者它卡在加载时，会让我的 Google Colab 没有响应。

用于 PCA

用于TSNE

我使用此源代码作为我的指南来执行 Tensorboard 嵌入可视化 https://medium.com/@kumon/visualizing-image-feature-vectors-through-tensorboard-b850ce1be7f1 但他使用 TensorFlow 和预训练模型 MobileNet。

我正在使用 Pytorch，所以我修改了网站上的编码以在 Pytorch 中使用。我使用的模型也是一个自定义模型，即 Siamese Network，它已经过训练，可以嵌入特征图像维度 250 并保存为权重模型路径。

在网站上，投影仪需要的是 sprite.jpg、feature_vecs.tsv 和 projector-config.pbtxt，以便投影仪可以工作。

我用这段代码来获取精灵.jpg。我的自定义数据集位于包含 14056 张图像的 rarfile 中。

import os
import rarfile
from PIL import Image
import torch
from torchvision import transforms
from torchvision.utils import make_grid

# Path to your rar file
rar_file_path = '/content/drive/MyDrive/DATASET-V1.rar'

# Directory to extract the images
extracted_folder = '/content/extracted_images/'

# Create a folder to extract the images if it doesn't exist
os.makedirs(extracted_folder, exist_ok=True)

# Extract the contents of the rar file
with rarfile.RarFile(rar_file_path, 'r') as rar_ref:
    rar_ref.extractall(extracted_folder)

# List all image files in the extracted directory including subfolders
image_files = []
for root, dirs, files in os.walk(extracted_folder):
    for file in files:
        if file.lower().endswith(('.png', '.jpg', '.jpeg', '.bmp')):
            image_files.append(os.path.join(root, file))

if len(image_files) == 0:
    print("No image files found in the extracted directory. Check the file extensions or the content of the extracted folder.")
else:
    # Define the number of images per row in the sprite image
    num_images_per_row = 100
    image_size = 50

    # Load and process the images using torchvision transforms
    transform = transforms.Compose([
        transforms.Resize((image_size, image_size)),
        transforms.ToTensor(),
    ])

    # Load and transform images
    images = []
    for image_file in image_files:
        try:
            img = Image.open(image_file).convert("RGB")
            img = transform(img)
            images.append(img)
        except Exception as e:
            print(f"Error loading {image_file}: {e}")

    if len(images) > 0:
        # Create a grid of images
        grid = make_grid(images, nrow=num_images_per_row)

        # Convert the PyTorch tensor to a PIL image
        sprite_image = transforms.ToPILImage()(grid)

        # Save the sprite image
        sprite_image.save('sprite.jpg')
    else:
        print("No images loaded successfully. Check for errors in image loading.")

        print("No images loaded successfully. Check for errors in image loading.")

接下来是我用来获取 feature_vecs.tsv 的代码。

import os
import torch
import torchvision.transforms as transforms
from torch.utils.data import DataLoader, Dataset
from PIL import Image
import numpy as np
import torch.nn as nn
import onnxruntime as ort
import rarfile
from io import BytesIO

# Path to your rar file
rar_file_path = '/content/drive/MyDrive/DATASET-V1.rar'

# Directory to extract the images
extracted_folder = '/content/extracted_images/'

# Create a folder to extract the images if it doesn't exist
os.makedirs(extracted_folder, exist_ok=True)

# Extract the contents of the rar file
with rarfile.RarFile(rar_file_path, 'r') as rar_ref:
    rar_ref.extractall(extracted_folder)

# Define the path to your Siamese network's encoder and optimizer weights
encoder_weight_path = "/content/drive/MyDrive/test_o/siam2/enc_weight_20231109022242.pth"
optimizer_weight_path = "/content/drive/MyDrive/test_o/siam2/optim_weight_20231110073800.pth"

# Initialize the Siamese network model and load the weights
class SiameseNetwork(nn.Module):

    def __init__(self):
        super(SiameseNetwork, self).__init__()

        # Setting up the Sequential of CNN Layers
        self.cnn1 = nn.Sequential(
            nn.Conv2d(3, 96, kernel_size=11,stride=4),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(3, stride=2),

            nn.Conv2d(96, 256, kernel_size=5, stride=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2, stride=2),

            nn.Conv2d(256, 384, kernel_size=3,stride=1),
            nn.ReLU(inplace=True)
        )

        # Setting up the Fully Connected Layers
        self.fc1 = nn.Sequential(
            nn.Linear(384*11*11, 1024),
            nn.ReLU(inplace=True),

            nn.Linear(1024, 256),
            nn.ReLU(inplace=True),

            nn.Linear(256,250)
        )

    def forward_once(self, x):
        # This function will be called for both images
        # It's output is used to determine the similiarity
        output = self.cnn1(x)
        output = output.view(output.size()[0], -1)
        output = self.fc1(output)
        return output


siamese_model = SiameseNetwork()  # Replace with the Siamese network model

# Load the encoder and optimizer weights
siamese_model.load_state_dict(torch.load(encoder_weight_path, map_location=torch.device('cpu')))
#siamese_model.load_state_dict(torch.load(optimizer_weight_path,  map_location=torch.device('cpu') ))

# Load the weights to the Siamese model
#siamese_model.load_state_dict(encoder_weights)
#siamese_model.load_state_dict(optimizer_weights)

# Define the preprocessing transformations
transform = transforms.Compose([
    transforms.Resize((256, 256)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])

# Define a custom dataset to load and preprocess the images from the extracted folder
class CustomDataset(Dataset):
    def __init__(self, image_dir, transform=None):
        self.image_dir = image_dir
        self.image_files = []
        for root, dirs, files in os.walk(image_dir):
            for file in files:
                if file.lower().endswith(('.png', '.jpg', '.jpeg', '.bmp')):
                    self.image_files.append(os.path.join(root, file))
        self.transform = transform

    def __len__(self):
        return len(self.image_files)

    def __getitem__(self, idx):
        img_path = self.image_files[idx]
        image = Image.open(img_path).convert("RGB")
        if self.transform:
            image = self.transform(image)
        return image

# Create a DataLoader for the dataset
dataset = CustomDataset(extracted_folder, transform)
dataloader = DataLoader(dataset, batch_size=10, shuffle=False)

# Initialize a list to store the extracted features
features = []

# Extract features for each batch of images using the Siamese network's encoder
for batch in dataloader:
    # Process a single batch of inputs
    input_data = batch.numpy()  # Convert PyTorch tensor to NumPy array

    # Forward pass to get the features
    with torch.no_grad():
        features_batch = siamese_model.forward_once(torch.tensor(input_data))

    # Convert PyTorch tensor to NumPy array
    features_batch = features_batch.cpu().numpy()

    # Append the features to the list
    features.append(features_batch)

# Concatenate the features along the batch dimension
features = np.concatenate(features)

# Save the extracted features to a file (e.g., feature_vecs2.tsv)
with open('feature_vecs2.tsv', 'w') as fw:
    np.savetxt(fw, features, delimiter='\t')

接下来是配置文件。对于我的 projector_config.pbtxt，我对其进行了编辑，使其具有精灵.jpg并具有 _vecs2.tsv。我将文件上传到 google colab。

投影仪配置文件

因此，我拥有所有 3 个文件。我用它来加载到 tensorboard 中。我在 Google Colab 中运行编码 我使用的 Tensorbord 代码是：

%load_ext tensorboard
%tensorboard --logdir /content

用于 PCA

用于TSNE

我希望从可视化中获得 10 个图像团块，但它显示所有图像组合成一个大球体。我不确定我是否以正确的方式在 pytorch 中使用 tensorboard，因为我是 pytorch 和 tensorboard 的新手。我想知道我需要在哪里修改代码才能使可视化效果在团块中显示相似的图像。有谁知道如何使用 tensorboard 投影图像嵌入以使用 pytorch 检查图像相似度准确性？