FreeU Random or Sequence Move for Animation

[zeittresor]

Hello, i had the idea to add a random move option to the BuildIn Extension FreeU.
(The script is not working correct in current state !)

The Idea is to change the value of the S1 and S2 Sliders randomly with +0.01 or -0.01 to the last S1 or S2 value - each time a Image in a Batch was generated. This make only sense if you have selected a static seed because the only thing what should change is "something in the Image itself".

This way every image is looking a little bit different "somehow like a morph". Adding all the Images using FFMPEG for example creates a nice effect (i have tried this manually but might be much more comfortable to have a checkbox to change the value automaticly).

Example:

My current Extension Ui:

My skills in python are currently not the best to get it working.. my personal current modified version of the FreeU Extension shows the checkbox but comes up with a lot of errors.. how ever if someone like the Idea please please try to help:

import torch
import gradio as gr
import random

from modules import scripts
from modules.script_callbacks import on_cfg_denoiser, remove_current_script_callbacks
from modules.ui_components import InputAccordion

def Fourier_filter(x, threshold, scale):
    # FFT
    x_freq = torch.fft.fftn(x.float(), dim=(-2, -1))
    x_freq = torch.fft.fftshift(x_freq, dim=(-2, -1))

    B, C, H, W = x_freq.shape
    mask = torch.ones((B, C, H, W), device=x.device)

    crow, ccol = H // 2, W // 2
    mask[..., crow - threshold:crow + threshold, ccol - threshold:ccl + threshold] = scale
    x_freq = x_freq * mask

    # IFFT
    x_freq = torch.fft.ifftshift(x_freq, dim=(-2, -1))
    x_filtered = torch.fft.ifftn(x_freq, dim=(-2, -1)).real

    return x_filtered.to(x.dtype)


def patch_freeu_v2(unet_patcher, b1, b2, s1, s2):
    model_channels = unet_patcher.model.diffusion_model.config.get("model_channels")

    scale_dict = {model_channels * 4: (b1, s1), model_channels * 2: (b2, s2)}
    on_cpu_devices = {}

    def output_block_patch(h, hsp, transformer_options):
        process = FreeUForForge.doFreeU

        if process:
            scale = scale_dict.get(h.shape[1], None)
            if scale is not None:
                hidden_mean = h.mean(1).unsqueeze(1)
                B = hidden_mean.shape[0]
                hidden_max, _ = torch.max(hidden_mean.view(B, -1), dim=-1, keepdim=True)
                hidden_min, _ = torch.min(hidden_mean.view(B, -1), dim=-1, keepdim=True)
                hidden_mean = (hidden_mean - hidden_min.unsqueeze(2).unsqueeze(3)) / (hidden_max - hidden_min).unsqueeze(2).unsqueeze(3)

                h[:, :h.shape[1] // 2] = h[:, :h.shape[1] // 2] * ((scale[0] - 1) * hidden_mean + 1)

                if hsp.device not in on_cpu_devices:
                    try:
                        hsp = Fourier_filter(hsp, threshold=1, scale=scale[1])
                    except:
                        print("Device", hsp.device, "does not support the torch.fft functions used in the FreeU node, switching to CPU.")
                        on_cpu_devices[hsp.device] = True
                        hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device)
                else:
                    hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device)

        return h, hsp

    m = unet_patcher.clone()
    m.set_model_output_block_patch(output_block_patch)
    return m


class FreeUForForge(scripts.Script):
    sorting_priority = 12  # It will be the 12th item on UI.
    
    doFreeU = True
    random_move_enabled = False  # Variable to control random UNet movement
    
    last_s1 = 0.99  # Store last values of S1 and S2
    last_s2 = 0.95

    presets_builtin = [
        #   name, b1, b2, s1, s2, start step, end step
        ('Forge default', 1.01, 1.02, 0.99, 0.95, 0.0, 1.0),
        ('SD 1.4', 1.3, 1.4, 0.9, 0.2, 0.0, 1.0),
        ('SD 1.5', 1.5, 1.6, 0.9, 0.2, 0.0, 1.0),
        ('SD 2.1', 1.4, 1.6, 0.9, 0.2, 0.0, 1.0),
        ('SDXL', 1.3, 1.4, 0.9, 0.2, 0.0, 1.0),
    ]
    try:
        import freeu_presets
        presets = presets_builtin + freeu_presets.presets_custom
    except:
        presets = presets_builtin

    def title(self):
        return "FreeU Integrated (SD 1.x, SD 2.x, SDXL)"

    def show(self, is_img2img):
        # make this extension visible in both txt2img and img2img tab.
        return scripts.AlwaysVisible

    def ui(self, *args, **kwargs):
        
        with InputAccordion(False, label=self.title(),
                          elem_id="extensions-freeu",
                          elem_classes=["extensions-freeu"]) as freeu_enabled:
            with gr.Row():
                freeu_b1 = gr.Slider(label='B1', minimum=0, maximum=2, step=0.01, value=1.01)
                freeu_b2 = gr.Slider(label='B2', minimum=0, maximum=2, step=0.01, value=1.02)
            with gr.Row():
                freeu_s1 = gr.Slider(label='S1', minimum=0, maximum=4, step=0.01, value=0.99)
                freeu_s2 = gr.Slider(label='S2', minimum=0, maximum=4, step=0.01, value=0.95)
            with gr.Row():
                freeu_start = gr.Slider(label='Start step', minimum=0.0, maximum=1.0, step=0.01, value=0.0)
                freeu_end   = gr.Slider(label='End step', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
            with gr.Row():
                freeu_preset = gr.Dropdown(label='', choices=[x[0] for x in FreeUForForge.presets], value='(presets)', type='index', scale=0, allow_custom_value=True)
            with gr.Row():
                random_move_checkbox = gr.Checkbox(label="Random UNet Move", value=False)

        def setParams (preset):
            if preset < len(FreeUForForge.presets):
                return  FreeUForForge.presets[preset][1], FreeUForForge.presets[preset][2], \
                        FreeUForForge.presets[preset][3], FreeUForForge.presets[preset][4], \
                        FreeUForForge.presets[preset][5], FreeUForForge.presets[preset][6], '(presets)'
            else:
                return 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, '(presets)'

        freeu_preset.input( fn=setParams,
                            inputs=[freeu_preset],
                            outputs=[freeu_b1, freeu_b2, freeu_s1, freeu_s2, freeu_start, freeu_end, freeu_preset], show_progress=False)

        self.infotext_fields = [
            (freeu_enabled, lambda d: d.get("freeu_enabled", False)),
            (freeu_b1,      "freeu_b1"),
            (freeu_b2,      "freeu_b2"),
            (freeu_s1,      "freeu_s1"),
            (freeu_s2,      "freeu_s2"),
            (freeu_start,   "freeu_start"),
            (freeu_end,     "freeu_end"),
            (random_move_checkbox, lambda d: d.get("random_move_enabled", False)),
        ]

        return freeu_enabled, freeu_b1, freeu_b2, freeu_s1, freeu_s2, freeu_start, freeu_end, random_move_checkbox

    def denoiser_callback(self, params):
        thisStep = params.sampling_step / (params.total_sampling_steps - 1)
        
        if thisStep >= FreeUForForge.freeu_start and thisStep <= FreeUForForge.freeu_end:
            FreeUForForge.doFreeU = True
        else:
            FreeUForForge.doFreeU = False

    def process_before_every_sampling(self, p, *script_args, **kwargs):
        # If you use highres fix, this will be called twice.

        freeu_enabled, freeu_b1, freeu_b2, freeu_s1, freeu_s2, freeu_start, freeu_end, random_move_checkbox = script_args

        if not freeu_enabled:
            return

        FreeUForForge.random_move_enabled = random_move_checkbox

        # Randomly adjust S1 and S2 if the option is enabled
        if FreeUForForge.random_move_enabled:
            FreeUForForge.last_s1 += random.choice([-0.01, 0.01])
            FreeUForForge.last_s2 += random.choice([-0.01, 0.01])

            # Clamp values to avoid going out of the allowed range (0 to 4)
            FreeUForForge.last_s1 = max(0, min(4, FreeUForForge.last_s1))
            FreeUForForge.last_s2 = max(0, min(4, FreeUForForge.last_s2))

            freeu_s1 = FreeUForForge.last_s1
            freeu_s2 = FreeUForForge.last_s2

        unet = p.sd_model.forge_objects.unet

        #   test if patchable
        model_channels = unet.model.diffusion_model.config.get("model_channels")
        if model_channels is None:
            gr.Info("FreeU is not supported for this model!")
            return

        FreeUForForge.freeu_start = freeu_start
        FreeUForForge.freeu_end   = freeu_end
        on_cfg_denoiser(self.denoiser_callback)

        unet = patch_freeu_v2(unet, freeu_b1, freeu_b2, freeu_s1, freeu_s2)

        p.sd_model.forge_objects.unet = unet

        # Below codes will add some logs to the texts below the image outputs on UI.
        p.extra_generation_params.update(dict(
            freeu_enabled   = freeu_enabled,
            freeu_b1        = freeu_b1,
            freeu_b2        = freeu_b2,
            freeu_s1        = freeu_s1,
            freeu_s2        = freeu_s2,
            freeu_start     = freeu_start,
            freeu_end       = freeu_end,
            random_move_enabled = FreeUForForge.random_move_enabled,
        ))

        return

    def postprocess(self, params, processed, *args):
        remove_current_script_callbacks()
        return

[DenOfEquity]

Interesting idea.
The only real error is the missing character in the Fourier_filter function. Line 18, you try to use variable ccl, should be ccol.

[zeittresor]

Oh, ok 👍 i will try this.. thanks..
btw. if someone want to use the idea feel free to do it.. ;-)

[zeittresor]

Ok i've changed the line "mask[..., crow - threshold:crow + threshold, ccol - threshold:ccl + threshold] = scale" to "mask[..., crow - threshold:crow + threshold, ccol - threshold:ccol + threshold] = scale" -> working.. (somehow) BUT i've a new problem or find a bug.

i tried to use it i first noticeing that the results are terrible against manually set changes to S1 / S2. I first did not understand why it is always generating stupid crap; But there seams to be a different "bug" or "problem" ...

if i enable the buildin freeU extension (with or without my new modifications): if i set the seed to a fixed value for ex. 1687053679 in a batch of 50 each new generated image also seams to add a seed=seed+1 to the fixed seed - any ideas where / why ?!

i mean it is not usual for someone to generate a single same looking image 50 times (by using the same seed) but for this extension it is required... i mean i can fix it in the extension by adding just "-1" to each new generating but that whould be a stupid dirty workaround for a bug somewhere else, its makeing more sense for me to find a solution for the increasing seed problem.

Test Output:
{"prompt": "a gentleman is standing next to a hotel bar waiting for a cocktail", "all_prompts": ["a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail", "a gentleman is standing next to a hotel bar waiting for a cocktail"], "negative_prompt": "", "all_negative_prompts": ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""], "seed": 1456784561, "all_seeds": [1456784561, 1456784562, 1456784563, 1456784564, 1456784565, 1456784566, 1456784567, 1456784568, 1456784569, 1456784570, 1456784571, 1456784572, 1456784573, 1456784574, 1456784575, 1456784576, 1456784577, 1456784578, 1456784579, 1456784580, 1456784581, 1456784582, 1456784583, 1456784584, 1456784585, 1456784586, 1456784587, 1456784588, 1456784589, 1456784590, 1456784591, 1456784592, 1456784593, 1456784594, 1456784595, 1456784596, 1456784597, 1456784598, 1456784599, 1456784600, 1456784601, 1456784602, 1456784603, 1456784604, 1456784605, 1456784606, 1456784607, 1456784608, 1456784609, 1456784610], "subseed": 663935569, "all_subseeds": [663935569, 663935570, 663935571, 663935572, 663935573, 663935574, 663935575, 663935576, 663935577, 663935578, 663935579, 663935580, 663935581, 663935582, 663935583, 663935584, 663935585, 663935586, 663935587, 663935588, 663935589, 663935590, 663935591, 663935592, 663935593, 663935594, 663935595, 663935596, 663935597, 663935598, 663935599, 663935600, 663935601, 663935602, 663935603, 663935604, 663935605, 663935606, 663935607, 663935608, 663935609, 663935610, 663935611, 663935612, 663935613, 663935614, 663935615, 663935616, 663935617, 663935618], "subseed_strength": 0, "width": 512, "height": 512, "sampler_name": "DDIM", "cfg_scale": 7, "steps": 20, "batch_size": 1, "restore_faces": true, "face_restoration_model": "CodeFormer", "sd_model_name": "NCHS-2", "sd_model_hash": "72f22f69b9", "sd_vae_name": null, "sd_vae_hash": null, "seed_resize_from_w": -1, "seed_resize_from_h": -1, "denoising_strength": 0.7, "extra_generation_params": {"Template": "a gentleman is standing next to a hotel bar waiting for a cocktail", "freeu_enabled": true, "freeu_b1": 1.5, "freeu_b2": 1.6, "freeu_s1": 1.01, "freeu_s2": 0.95, "freeu_start": 0, "freeu_end": 1, "random_move_enabled": true}, "index_of_first_image": 0, "infotexts": ["a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784561, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 0.99, freeu_s2: 0.95, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784562, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 1.0, freeu_s2: 0.94, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784563, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 0.99, freeu_s2: 0.9299999999999999, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784564, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 1.0, freeu_s2: 0.9199999999999999, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784565, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 0.99, freeu_s2: 0.9299999999999999, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784566, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 0.98, freeu_s2: 0.94, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784567, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 0.99, freeu_s2: 0.95, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784568, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 1.0, freeu_s2: 0.96, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784569, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 0.99, freeu_s2: 0.97, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784570, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 1.0, freeu_s2: 0.96, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20", "a gentleman is standing next to a hotel bar waiting for a cocktail\nSteps: 20, Sampler: DDIM, Schedule type: Automatic, CFG scale: 7, Seed: 1456784571, Face restoration: CodeFormer, Size: 512x512, Model hash: 72f22f69b9, Model: NCHS-2, Clip skip: 3, Template: a gentleman is standing next to a hotel bar waiting for a cocktail, freeu_enabled: True, freeu_b1: 1.5, freeu_b2: 1.6, freeu_s1: 1.01, freeu_s2: 0.95, freeu_start: 0, freeu_end: 1, random_move_enabled: True, Version: f2.0.1v1.10.1-previous-540-gc2d290e6, Module 1: wd.vae, Module 2: zVae_v20"], "styles": [], "job_timestamp": "20240925181641", "clip_skip": 3, "is_using_inpainting_conditioning": false, "version": "f2.0.1v1.10.1-previous-540-gc2d290e6"}

[zeittresor]

btw. if i change the S1 Value without a Batch doing this manually it still works like expected..

example2.mp4

[zeittresor]

might be a major problem, even with setting freeU to disabled while setting:

..but its increasing the seed also. even if i change this to disabled it always adds "+1" to the seed in a batch:

Someone any idea how to fix it ? (how ever could also by a problem with extensions but most of them i am using are not enabled)..

[zeittresor]

So far the only working solution seems to use something like this

First result by using the "random unet move" option (its a little bit unstable because plain randomly changes the unet move without a stepping sequence like s1+0.01/s1+0.01/s1+0.01/s2-0.01/s1+0.01/s1+0.01/s1+0.01/s2-0.01/s1+0.01/s1+0.01/s1+0.01/s2-0.01/.. result in something like this)..
https://github.com/user-attachments/assets/2cebb661-df71-4037-87ec-f26493e6b6f3

Thanks to Haoming02 for the tip using the x/y/z script in #1910

[DenOfEquity]

this seems like a case for 'Generate Forever', right-click the Generate button. Pick a seed (not -1), batch 1 only, no script needed, Forge browser tab needs to stay active.
That 'Use same seed for all images' setting doesn't exist for me, maybe it is added by an extension?

If you haven't already, you could also try smaller adjustments, maybe with more choices. Something like random.choice([-0.002, -0.001, 0.001, 0.002]).

[zeittresor]

this seems like a case for 'Generate Forever', right-click the Generate button. Pick a seed (not -1), batch 1 only, no script needed, Forge browser tab needs to stay active. That 'Use same seed for all images' setting doesn't exist for me, maybe it is added by an extension?

If you haven't already, you could also try smaller adjustments, maybe with more choices. Something like random.choice([-0.002, -0.001, 0.001, 0.002]).

Thanks a lot i will test the generate forever option.. i have reworked the script right now again by a adding just another checkbox to move the S1 value .. here is a first test..

Result:

[DenOfEquity]

seems like that results in smoother transitions - works well

[zeittresor]

this seems like a case for 'Generate Forever', right-click the Generate button. Pick a seed (not -1), batch 1 only, no script needed, Forge browser tab needs to stay active. That 'Use same seed for all images' setting doesn't exist for me, maybe it is added by an extension?

If you haven't already, you could also try smaller adjustments, maybe with more choices. Something like random.choice([-0.002, -0.001, 0.001, 0.002]).

hi, btw. it works very well with the right click on the generate button (generate forever) 👍

[zeittresor]

here is the code for the \webui\extensions-builtin\sd_forge_freeu\scripts replacement of forge_freeu.py to add this:

`
import torch
import gradio as gr
import random

from modules import scripts
from modules.script_callbacks import on_cfg_denoiser, remove_current_script_callbacks
from modules.ui_components import InputAccordion

def Fourier_filter(x, threshold, scale):
    # FFT
    x_freq = torch.fft.fftn(x.float(), dim=(-2, -1))
    x_freq = torch.fft.fftshift(x_freq, dim=(-2, -1))

    B, C, H, W = x_freq.shape
    mask = torch.ones((B, C, H, W), device=x.device)

    crow, ccol = H // 2, W // 2
    mask[..., crow - threshold:crow + threshold, ccol - threshold:ccol + threshold] = scale
    x_freq = x_freq * mask

    # IFFT
    x_freq = torch.fft.ifftshift(x_freq, dim=(-2, -1))
    x_filtered = torch.fft.ifftn(x_freq, dim=(-2, -1)).real

    return x_filtered.to(x.dtype)


def patch_freeu_v2(unet_patcher, b1, b2, s1, s2):
    model_channels = unet_patcher.model.diffusion_model.config.get("model_channels")

    scale_dict = {model_channels * 4: (b1, s1), model_channels * 2: (b2, s2)}
    on_cpu_devices = {}

    def output_block_patch(h, hsp, transformer_options):
        process = FreeUForForge.doFreeU

        if process:
            scale = scale_dict.get(h.shape[1], None)
            if scale is not None:
                hidden_mean = h.mean(1).unsqueeze(1)
                B = hidden_mean.shape[0]
                hidden_max, _ = torch.max(hidden_mean.view(B, -1), dim=-1, keepdim=True)
                hidden_min, _ = torch.min(hidden_mean.view(B, -1), dim=-1, keepdim=True)
                hidden_mean = (hidden_mean - hidden_min.unsqueeze(2).unsqueeze(3)) / (hidden_max - hidden_min).unsqueeze(2).unsqueeze(3)

                h[:, :h.shape[1] // 2] = h[:, :h.shape[1] // 2] * ((scale[0] - 1) * hidden_mean + 1)

                if hsp.device not in on_cpu_devices:
                    try:
                        hsp = Fourier_filter(hsp, threshold=1, scale=scale[1])
                    except:
                        print("Device", hsp.device, "does not support the torch.fft functions used in the FreeU node, switching to CPU.")
                        on_cpu_devices[hsp.device] = True
                        hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device)
                else:
                    hsp = Fourier_filter(hsp.cpu(), threshold=1, scale=scale[1]).to(hsp.device)

        return h, hsp

    m = unet_patcher.clone()
    m.set_model_output_block_patch(output_block_patch)
    return m


class FreeUForForge(scripts.Script):
    sorting_priority = 12  # It will be the 12th item on UI.
    
    doFreeU = True
    random_move_enabled = False  # Variable to control random UNet movement
    simple_move_s1_enabled = False # Variable to control S1 UNet movement
    
    last_s1 = 0.99  # Store last values of S1 and S2
    last_s2 = 0.95

    presets_builtin = [
        #   name, b1, b2, s1, s2, start step, end step
        ('Forge default', 1.01, 1.02, 0.99, 0.95, 0.0, 1.0),
        ('SD 1.4', 1.3, 1.4, 0.9, 0.2, 0.0, 1.0),
        ('SD 1.5', 1.5, 1.6, 0.9, 0.2, 0.0, 1.0),
        ('SD 2.1', 1.4, 1.6, 0.9, 0.2, 0.0, 1.0),
        ('SDXL', 1.3, 1.4, 0.9, 0.2, 0.0, 1.0),
    ]
    try:
        import freeu_presets
        presets = presets_builtin + freeu_presets.presets_custom
    except:
        presets = presets_builtin

    def title(self):
        return "FreeU Integrated (SD 1.x, SD 2.x, SDXL)"

    def show(self, is_img2img):
        # make this extension visible in both txt2img and img2img tab.
        return scripts.AlwaysVisible

    def ui(self, *args, **kwargs):
        
        with InputAccordion(False, label=self.title(),
                          elem_id="extensions-freeu",
                          elem_classes=["extensions-freeu"]) as freeu_enabled:
            with gr.Row():
                freeu_b1 = gr.Slider(label='B1', minimum=0, maximum=2, step=0.01, value=1.01)
                freeu_b2 = gr.Slider(label='B2', minimum=0, maximum=2, step=0.01, value=1.02)
            with gr.Row():
                freeu_s1 = gr.Slider(label='S1', minimum=0, maximum=4, step=0.01, value=0.99)
                freeu_s2 = gr.Slider(label='S2', minimum=0, maximum=4, step=0.01, value=0.95)
            with gr.Row():
                freeu_start = gr.Slider(label='Start step', minimum=0.0, maximum=1.0, step=0.01, value=0.0)
                freeu_end   = gr.Slider(label='End step', minimum=0.0, maximum=1.0, step=0.01, value=1.0)
            with gr.Row():
                freeu_preset = gr.Dropdown(label='', choices=[x[0] for x in FreeUForForge.presets], value='(presets)', type='index', scale=0, allow_custom_value=True)
            with gr.Row():
                random_move_checkbox = gr.Checkbox(label="Random UNet Move", value=False)
            with gr.Row():
                simple_move_s1_checkbox = gr.Checkbox(label="Simple UNet S1 Move", value=False)

        def setParams (preset):
            if preset < len(FreeUForForge.presets):
                return  FreeUForForge.presets[preset][1], FreeUForForge.presets[preset][2], \
                        FreeUForForge.presets[preset][3], FreeUForForge.presets[preset][4], \
                        FreeUForForge.presets[preset][5], FreeUForForge.presets[preset][6], '(presets)'
            else:
                return 1.0, 1.0, 1.0, 1.0, 0.0, 1.0, '(presets)'

        freeu_preset.input( fn=setParams,
                            inputs=[freeu_preset],
                            outputs=[freeu_b1, freeu_b2, freeu_s1, freeu_s2, freeu_start, freeu_end, freeu_preset], show_progress=False)

        self.infotext_fields = [
            (freeu_enabled, lambda d: d.get("freeu_enabled", False)),
            (freeu_b1,      "freeu_b1"),
            (freeu_b2,      "freeu_b2"),
            (freeu_s1,      "freeu_s1"),
            (freeu_s2,      "freeu_s2"),
            (freeu_start,   "freeu_start"),
            (freeu_end,     "freeu_end"),
            (random_move_checkbox, lambda d: d.get("random_move_enabled", False)),
            (simple_move_s1_checkbox, lambda d: d.get("simple_move_s1_enabled", False)),
        ]

        return freeu_enabled, freeu_b1, freeu_b2, freeu_s1, freeu_s2, freeu_start, freeu_end, random_move_checkbox, simple_move_s1_checkbox

    def denoiser_callback(self, params):
        thisStep = params.sampling_step / (params.total_sampling_steps - 1)
        
        if thisStep >= FreeUForForge.freeu_start and thisStep <= FreeUForForge.freeu_end:
            FreeUForForge.doFreeU = True
        else:
            FreeUForForge.doFreeU = False

    def process_before_every_sampling(self, p, *script_args, **kwargs):
        # If you use highres fix, this will be called twice.

        freeu_enabled, freeu_b1, freeu_b2, freeu_s1, freeu_s2, freeu_start, freeu_end, random_move_checkbox, simple_move_s1_checkbox = script_args

        if not freeu_enabled:
            return

        FreeUForForge.random_move_enabled = random_move_checkbox
        FreeUForForge.simple_move_s1_enabled = simple_move_s1_checkbox

        # Randomly adjust S1 and S2 if the option is enabled
        if FreeUForForge.random_move_enabled:
            FreeUForForge.last_s1 += random.choice([-0.01, 0.01])
            FreeUForForge.last_s2 += random.choice([-0.01, 0.01])

            # Clamp values to avoid going out of the allowed range (0 to 4)
            FreeUForForge.last_s1 = max(0, min(4, FreeUForForge.last_s1))
            FreeUForForge.last_s2 = max(0, min(4, FreeUForForge.last_s2))

            freeu_s1 = FreeUForForge.last_s1
            freeu_s2 = FreeUForForge.last_s2

        unet = p.sd_model.forge_objects.unet

        # Simple increase S1 if the option is enabled
        if FreeUForForge.simple_move_s1_enabled:
            FreeUForForge.last_s1 += 0.01
            FreeUForForge.last_s2 = FreeUForForge.last_s2

            # Clamp values to avoid going out of the allowed range (0 to 4)
            FreeUForForge.last_s1 = max(0, min(4, FreeUForForge.last_s1))
            FreeUForForge.last_s2 = max(0, min(4, FreeUForForge.last_s2))

            freeu_s1 = FreeUForForge.last_s1
            freeu_s2 = FreeUForForge.last_s2

        unet = p.sd_model.forge_objects.unet

        #   test if patchable
        model_channels = unet.model.diffusion_model.config.get("model_channels")
        if model_channels is None:
            gr.Info("FreeU is not supported for this model!")
            return

        FreeUForForge.freeu_start = freeu_start
        FreeUForForge.freeu_end   = freeu_end
        on_cfg_denoiser(self.denoiser_callback)

        unet = patch_freeu_v2(unet, freeu_b1, freeu_b2, freeu_s1, freeu_s2)

        p.sd_model.forge_objects.unet = unet

        # Below codes will add some logs to the texts below the image outputs on UI.
        p.extra_generation_params.update(dict(
            freeu_enabled   = freeu_enabled,
            freeu_b1        = freeu_b1,
            freeu_b2        = freeu_b2,
            freeu_s1        = freeu_s1,
            freeu_s2        = freeu_s2,
            freeu_start     = freeu_start,
            freeu_end       = freeu_end,
            random_move_enabled = FreeUForForge.random_move_enabled,
            simple_move_s1_enabled = FreeUForForge.simple_move_s1_enabled,
        ))

        return

    def postprocess(self, params, processed, *args):
        remove_current_script_callbacks()
        return
`

[zeittresor]

can i add this as a "new" extension even if most of the original code is from the original "build-in" extension of forge_freeu.py or what should i do with the code ?

[DenOfEquity]

Just include a comment with attribution of the original in your code and any documentation. At the moment, your version will conflict with the built-in if installed at the same time. You could simply find/replace freeu with something else; and update the title.

[zeittresor]

Just include a comment with attribution of the original in your code and any documentation. At the moment, your version will conflict with the built-in if installed at the same time. You could simply find/replace freeu with something else; and update the title.

good idea thanks for your advice. 😊

[zeittresor]

btw. i've created another script to make a smoother transition between the "keyframes" its working fine so far but the mp4 output is broken, needs some more work i think 😎

[zeittresor]

# filename: createvid.py
import cv2 # pip install opencv-python
import numpy as np 
import os
import subprocess
from PIL import Image #pip install pillow
import argparse

# requires FFMPEG in the same folder like the script to not just create intermediate images but also a vid
# if no ffmpeg is found it would create the intermediate images but result with a error after that process.
# https://www.ffmpeg.org/download.html


def load_images_from_folder(folder):
    images = []
    filenames = []
    for filename in sorted(os.listdir(folder)):
        img = cv2.imread(os.path.join(folder, filename))
        if img is not None:
            images.append(img)
            filenames.append(filename)
    return images, filenames

def generate_intermediate_images(img1, img2, num_intermediates=3):
    intermediates = []
    for i in range(1, num_intermediates + 1):
        alpha = i / (num_intermediates + 1)
        intermediate = cv2.addWeighted(img1, 1 - alpha, img2, alpha, 0)
        intermediates.append(intermediate)
    return intermediates

def save_images(images, base_filename, output_folder):
    base_name, ext = os.path.splitext(base_filename)
    for idx, img in enumerate(images):
        output_filename = f"{base_name}_{idx + 1}{ext}"
        cv2.imwrite(os.path.join(output_folder, output_filename), img)

def create_video_from_images(folder, output_video, fps=30):
    subprocess.run([
        'ffmpeg', '-framerate', str(fps), '-i', os.path.join(folder, '%*.png'),
        '-c:v', 'libx264', '-pix_fmt', 'yuv420p', output_video
    ])

def upscale_image(image, scale_factor=2):
    pil_image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
    new_size = (pil_image.width * scale_factor, pil_image.height * scale_factor)
    upscaled_image = pil_image.resize(new_size, Image.LANCZOS)
    return cv2.cvtColor(np.array(upscaled_image), cv2.COLOR_RGB2BGR)

def resize_images(images, size):
    resized_images = []
    for img in images:
        resized_img = cv2.resize(img, size)
        resized_images.append(resized_img)
    return resized_images

def ensure_three_channels(images):
    converted_images = []
    for img in images:
        if len(img.shape) == 2:
            img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
        elif img.shape[2] == 4:
            img = cv2.cvtColor(img, cv2.COLOR_BGRA2BGR)
        converted_images.append(img)
    return converted_images

def main(folder, num_intermediates=3, output_video='output.mp4', upscale=False):
    images, filenames = load_images_from_folder(folder)
    if upscale:
        images = [upscale_image(img) for img in images]
    
    images = ensure_three_channels(images)
    
    if images:
        size = (images[0].shape[1], images[0].shape[0])
        images = resize_images(images, size)
    
    for i in range(len(images) - 1):
        intermediates = generate_intermediate_images(images[i], images[i + 1], num_intermediates)
        save_images(intermediates, filenames[i], folder)
    create_video_from_images(folder, output_video)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Create Vid from images for unet move animation by Zeittresor')
    parser.add_argument('folder', type=str, help='Folder containing images ex. python createvid.py . to use the current folder')
    parser.add_argument('--num_intermediates', type=int, default=3, help='Number of intermediate images to generate (default: 3)')
    parser.add_argument('--output_video', type=str, default='output.mp4', help='Output video file name (default: output.mp4)')
    parser.add_argument('--upscale', action='store_true', help='Upscale images by 2x using lanczos before processing (set this to scale images up)')
    args = parser.parse_args()
    main(args.folder, args.num_intermediates, args.output_video, args.upscale)

```
`

[zeittresor]

finaly its here https://github.com/zeittresor/sd-forge-fum thanks everybody for the tips 🥇

[zeittresor]

Hi, Update FYI 😉 i have modified my timedviewer script (https://github.com/zeittresor/timedviewer) with a preset to watch the video generation progress in realtime while the sd-forge-fum extension is running..