Copy_of_FarStrike_Sonnet.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TheOneTrueGuy/DigitalDayDream/blob/main/Copy_of_FarStrike_Sonnet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_2gnooYgJmIt"
      },
      "outputs": [],
      "source": [
        "# What do these installs tell us about the code we are going to be writing?\n",
        "# How will the functions and operation depend on these dependencies?\n",
        "# What methods, variables, objects and function from these libraries will be used to achieve these ends?\n",
        "# watch closely for the imports and import from statements that call on these libraries.\n",
        "!pip install git+https://github.com/huggingface/diffusers.git\n",
        "!pip install --upgrade accelerate #transformers\n",
        "!pip install omegaconf\n",
        "!apt-get install rsync # for better file transfer progress?\n",
        "!pip install noise\n",
        "!pip install anthropic # for Claude Sonnet\n",
        "# These libraries will be used to generate image-to-image diffusion model outputs specifically an image.\n",
        "# More specifically, diffusers is a library for training diffusion models, generating images\n",
        "# accelerate is a library for distributed inference, and transformers is a library for natural language processing.\n",
        "# and this package (notebook) as a whole is used to generate frames of an animation using diffusion models, huggingface pipelines and imagemagick convert command\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git lfs install\n",
        "!git clone https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers\n",
        "#"
      ],
      "metadata": {
        "id": "hVKtC7mAat7n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p43hYzDe0nXC"
      },
      "outputs": [],
      "source": [
        "#!pip install omegaconf #for use with deliberate\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "#!mkdir 8_1_2023\n",
        "!mkdir src\n",
        "\n",
        "#!cp /content/drive/MyDrive/stub_materials/p17.png  /content/st.png # cosmicconnection\n",
        "!cp /content/drive/MyDrive/stub_materials/bio10_12_23.txt /content/\n",
        "!cp /content/drive/MyDrive/stub_materials/building.txt /content/\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NBPvWwi2M29h"
      },
      "outputs": [],
      "source": [
        "!apt-get update\n",
        "!apt install imagemagick\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xcljrDfeoyla"
      },
      "outputs": [],
      "source": [
        "#@title upload start image 512x512 will be quick and look good\n",
        "\n",
        "import os\n",
        "\n",
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "name=uploaded.keys()\n",
        "name = list(name)[0]\n",
        "print(name)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AUODCru8OG0-"
      },
      "outputs": [],
      "source": [
        "\n",
        "os.system(f\"convert {name} -resize 512x512! st.png\")\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import anthropic\n",
        "\n",
        "client = anthropic.Anthropic(\n",
        "    # defaults to os.environ.get(\"ANTHROPIC_API_KEY\")\n",
        "    api_key=\"your anthropic key here\",\n",
        ")\n",
        "\n",
        "def generate_text(prompt):\n",
        "    message = client.messages.create(\n",
        "        model=\"claude-3-5-sonnet-20240620\",\n",
        "        max_tokens=1024,\n",
        "        messages=[\n",
        "            {\"role\": \"user\", \"content\": prompt}\n",
        "        ]\n",
        "    )\n",
        "    print(message.content)\n",
        "    return message.content\n",
        "\n"
      ],
      "metadata": {
        "id": "c9GnSNY7tnDa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import anthropic\n",
        "from PIL import Image\n",
        "import base64\n",
        "import io\n",
        "\n",
        "def load_and_encode_image(image_path, media_type):\n",
        "    with Image.open(image_path) as img:\n",
        "        img_byte_arr = io.BytesIO()\n",
        "        img.save(img_byte_arr, format=media_type.split('/')[-1].upper())\n",
        "        img_byte_arr = img_byte_arr.getvalue()\n",
        "        image_data = base64.b64encode(img_byte_arr).decode(\"utf-8\")\n",
        "    return {\n",
        "        \"data\": image_data,\n",
        "        \"media_type\": media_type\n",
        "    }\n",
        "\n",
        "def create_image_message(image_path, text_prompt):\n",
        "    # Determine media type based on file extension\n",
        "    media_type = f\"image/{image_path.split('.')[-1].lower()}\"\n",
        "    if media_type == \"image/jpg\":\n",
        "        media_type = \"image/jpeg\"\n",
        "\n",
        "    # Load and encode the image\n",
        "    image_info = load_and_encode_image(image_path, media_type)\n",
        "\n",
        "    # Set up the Anthropic client\n",
        "    client = anthropic.Anthropic()\n",
        "\n",
        "    # Create and return the message\n",
        "    return client.messages.create(\n",
        "        model=\"claude-3-5-sonnet-20240620\",\n",
        "        max_tokens=1024,\n",
        "        messages=[\n",
        "            {\n",
        "                \"role\": \"user\",\n",
        "                \"content\": [\n",
        "                    {\n",
        "                        \"type\": \"image\",\n",
        "                        \"source\": {\n",
        "                            \"type\": \"base64\",\n",
        "                            \"media_type\": image_info['media_type'],\n",
        "                            \"data\": image_info['data'],\n",
        "                        },\n",
        "                    },\n",
        "                    {\n",
        "                        \"type\": \"text\",\n",
        "                        \"text\": text_prompt\n",
        "                    }\n",
        "                ],\n",
        "            }\n",
        "        ],\n",
        "    )\n",
        "\n",
        "\n",
        "\n",
        "# Example usage\n",
        "image_path = \"st.png\"\n",
        "text_prompt = \"Describe this image.\"\n",
        "\n",
        "response = create_image_message(image_path, text_prompt)\n",
        "print(response)\n",
        "\n",
        "def image_to_message(img: Image.Image, text_prompt: str, media_type: str = None):\n",
        "    def encode_image(img: Image.Image, media_type: str):\n",
        "        img_byte_arr = io.BytesIO()\n",
        "        img.save(img_byte_arr, format=media_type.split('/')[-1].upper())\n",
        "        img_byte_arr = img_byte_arr.getvalue()\n",
        "        image_data = base64.b64encode(img_byte_arr).decode(\"utf-8\")\n",
        "        return {\n",
        "            \"data\": image_data,\n",
        "            \"media_type\": media_type\n",
        "        }\n",
        "\n",
        "    # Determine media type if not provided\n",
        "    if media_type is None:\n",
        "        if img.format == 'PNG':\n",
        "            media_type = 'image/png'\n",
        "        elif img.format in ['JPEG', 'JPG']:\n",
        "            media_type = 'image/jpeg'\n",
        "        else:\n",
        "            media_type = f'image/{img.format.lower()}'\n",
        "\n",
        "    # Encode the image\n",
        "    image_info = encode_image(img, media_type)\n",
        "\n",
        "    # Set up the Anthropic client\n",
        "    client = anthropic.Anthropic()\n",
        "\n",
        "    # Create and return the message\n",
        "    return client.messages.create(\n",
        "        model=\"claude-3-5-sonnet-20240620\",\n",
        "        max_tokens=1024,\n",
        "        messages=[\n",
        "            {\n",
        "                \"role\": \"user\",\n",
        "                \"content\": [\n",
        "                    {\n",
        "                        \"type\": \"image\",\n",
        "                        \"source\": {\n",
        "                            \"type\": \"base64\",\n",
        "                            \"media_type\": image_info['media_type'],\n",
        "                            \"data\": image_info['data'],\n",
        "                        },\n",
        "                    },\n",
        "                    {\n",
        "                        \"type\": \"text\",\n",
        "                        \"text\": text_prompt\n",
        "                    }\n",
        "                ],\n",
        "            }\n",
        "        ],\n",
        "    )"
      ],
      "metadata": {
        "id": "Iy0lTTIwzcIZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oYicA98RX_DZ"
      },
      "outputs": [],
      "source": [
        "\n",
        "import string\n",
        "import random\n",
        "from collections import deque\n",
        "import math\n",
        "\n",
        "\n",
        "class Img_Stack:\n",
        "    def __init__(self):\n",
        "        self.stack = deque(maxlen=20)\n",
        "        self.count = 0\n",
        "\n",
        "    def push(self, image1):\n",
        "        self.stack.append(image1)\n",
        "        self.count += 1\n",
        "        if self.count>19: self.count=19\n",
        "\n",
        "    def get_stack(self):\n",
        "        return list(self.stack)\n",
        "\n",
        "    def get_item(self, index):\n",
        "        if index >= 0 and index < len(self.stack):\n",
        "          return self.stack[index]\n",
        "        else:\n",
        "          return None\n",
        "\n",
        "stacky=Img_Stack()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a4O2mYM6Kld4"
      },
      "outputs": [],
      "source": [
        "# run this first then make anim in next cell\n",
        "\n",
        "import torch\n",
        "\n",
        "from diffusers.utils import load_image\n",
        "from transformers import AutoTokenizer # for use with small sd\n",
        "\n",
        "import datetime as dt\n",
        "import time\n",
        "import os\n",
        "import random\n",
        "from PIL import ImageFont, ImageDraw, Image, ImageOps, ImageChops, ImageFilter\n",
        "\n",
        "class FakeSafety():\n",
        "    def __call__(self, clip_input, images):\n",
        "        return (images, [False])\n",
        "\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "hf_auth=\"your huggingface auth key here\"\n",
        "\n",
        "from diffusers import StableDiffusion3Img2ImgPipeline\n",
        "\n",
        "\n",
        "pipe = StableDiffusion3Img2ImgPipeline.from_pretrained(\"stabilityai/stable-diffusion-3-medium-diffusers\",\n",
        "                                                torch_dtype=torch.float16, token=hf_auth,  text_encoder_3=None, tokenizer_3=None)\n",
        "\n",
        "pipe = pipe.to(\"cuda\")\n",
        "pipe.safety_checker = FakeSafety()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "code",
        "id": "Ec7ofJFQQAom"
      },
      "outputs": [],
      "source": [
        "import string #cell number 11!\n",
        "# subject description, background, additional elements, effect, overall composition\n",
        "\n",
        "# how about trying this with a procedural method loop?\n",
        "\n",
        "zip_name = \"06_25_24_farstrike_sonnet.zip\"  #@param {type:\"string\"}\n",
        "\n",
        "outdir = dt.datetime.now().strftime(\"%Y%m%d%H%M\")\n",
        "# if outdir exists add 4 random digits to it:\n",
        "if os.path.exists(outdir):\n",
        "    # Generate 4 random digits\n",
        "    random_digits = str(random.randint(1000, 9999))\n",
        "\n",
        "    # Create a new output directory with the random digits appended\n",
        "    new_outdir = outdir + random_digits\n",
        "\n",
        "    os.mkdir(new_outdir)\n",
        "    outdir=new_outdir\n",
        "    print(\"Created new output directory:\", new_outdir)\n",
        "else:\n",
        "    os.system(f\"mkdir {outdir}\")\n",
        "print(\"date time now:\" + outdir)\n",
        "\n",
        "os.system(f\"cp st.png {outdir}\")\n",
        "style=\"\" # can add style to prompt later\n",
        "\n",
        "xl=0.31\n",
        "xr=0\n",
        "gs=7.21\n",
        "gd=.1\n",
        "ns=0.65\n",
        "xz=0\n",
        "nx=0\n",
        "\n",
        "\n",
        "total=200\n",
        "\n",
        "negprompt=\"\"\n",
        "\n",
        "initial_value = 1  # Initial value of C2 use for exact-ish file count and timing\n",
        "\n",
        "\n",
        "\n",
        "image3= load_image(\"st.png\").convert(\"RGB\")\n",
        "for xn in range(0,total,1):#must equal xz\n",
        "\n",
        "\n",
        "  if xn%50==49: torch.cuda.empty_cache()\n",
        "\n",
        "  negstyle=\"\"\n",
        "  negprompt=\"\"\n",
        "  #negstyle=\"dumb, flat, ordinary, monochrome, boring, sad, insipid, ordinary, human face\" # , unhappy, worried, angry , weird face, scarlet, distorted face, weird nose\" # human, person, face, head, hands, man, woman, child, \" # dispirated, terrorized,  malformed hands, misshapen hands,\n",
        "  negprompt = negprompt + negstyle #prefix with style\n",
        "  print(prompt)\n",
        "\n",
        "  init_image = load_image(\"st.png\").convert(\"RGB\")\n",
        "\n",
        "  instruction=\"\"\"\n",
        "  This image is a frame in an animation. The animation subject is nanotechnology. Using the formula:\n",
        "   subject description, background, additional elements, effect, overall composition\n",
        "   you will suggest a brief image prompt that would change this image into the next frame in the animation subject of nanotechnology. Reply only with:\n",
        "   subject description, background, additional elements, effect, overall composition\n",
        "  \"\"\"\n",
        "  prompt=image_to_message(init_image, instruction)\n",
        "\n",
        "  nsize=(512,512)\n",
        "  init_image=init_image.resize(nsize)\n",
        "  print(\"negative:\"+negprompt)\n",
        "\n",
        "  image2 = init_image\n",
        "  if xn>2:\n",
        "    image2=stacky.get_item(2)\n",
        "\n",
        "  image = pipe(\n",
        "               prompt,\n",
        "               num_inference_steps=28,\n",
        "               strength=0.5,\n",
        "               guidance_scale=gs, #11.721,\n",
        "               negative_prompt = negprompt,\n",
        "               image=init_image, # init_image,\n",
        "               ).images[0]\n",
        "  gs+=.006  #sr=sin_ratio(xn, 0.086)\n",
        "\n",
        "\n",
        "\n",
        "  stacky.push(image)\n",
        "  outname= \"st.png\"\n",
        "\n",
        "  out2=outdir+\"/st-\" + str(xz).zfill(4) + \".png\"\n",
        "  image.save(out2) # save animation frame\n",
        "  xz+=1\n",
        "  #out2=outdir+\"/st-\" + str(xz).zfill(4) + \".png\" # 3 lines added 2/14/24 to roll back remove/comment and change image.save above to imageb.save\n",
        "  #imageb.save(out2)\n",
        "  #xz+=1\n",
        "\n",
        "\n",
        "  image.save(outname)\n",
        "\n",
        "os.system(f\"cp 1julien3.sh {outdir}\")\n",
        "#os.system(f\"cp moto1g.txt {outdir}\")\n",
        "os.system(f\"zip -r {zip_name} {outdir}\")\n",
        "os.system(f\"cp {zip_name}  /content/drive/MyDrive/product\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cxW7uiWg5sc3"
      },
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X4yyeR7YPa4p"
      },
      "outputs": [],
      "source": [
        "#run this cell after starting cell above if you don't want to burn\n",
        "#google compute credits unnecessarily.\n",
        "print(dt.datetime.now().strftime(\"%Y%M%d%h%m\"))\n",
        "from google.colab import runtime\n",
        "runtime.unassign()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "machine_shape": "hm",
      "provenance": [],
      "authorship_tag": "ABX9TyNDZJ7tIZr2VbEPq1VjP5f2",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}