This week, building on last week's work, I mainly added two features: one is generating musical notes based on uploaded songs, and the second is creating a particle effect when a note is hit. In the following sections, I'll explain how I implemented these two features.

Link：https://editor.p5js.org/StonesGate604/sketches/jDtkEMNqM

// Week 11 Assignment Zekai Wang 
//generate sound notes from the music uploaded

let lanes = 4;
let laneWidth;
let hitLineY = 650;
let noteSpeed = 4;
let hitWindow = 50;

let notes = [];
let particles = [];
let oscillators = [];
let freqs = [261.63, 329.63, 392.0, 523.25];

let song;
let fft;
let peakDetect;
let lastSpawnTime = 0;
let spawnCooldown = 15;

let keyToLane = {
  D: 0,
  F: 1,
  J: 2,
  K: 3,
};

function preload() {
  song = loadSound('music.mp3');
}

function setup() {
  createCanvas(600, 800);
  laneWidth = width / lanes;

  for (let i = 0; i < lanes; i++) {
    let osc = new p5.Oscillator("sine");
    osc.freq(freqs[i]);
    osc.amp(0);
    osc.start();
    oscillators.push(osc);
  }

  fft = new p5.FFT();
  peakDetect = new p5.PeakDetect(20, 20000, 0.15, 20);

  textAlign(CENTER, CENTER);
  textSize(24);
}

function draw() {
  background(30);

  if (!song.isPlaying()) {
    fill(255);
    text("Click to Start Music", width/2, height/2);
    return;
  }

  fft.analyze();
  peakDetect.update(fft);

  if (peakDetect.isDetected && frameCount > lastSpawnTime + spawnCooldown) {
    spawnNoteByFrequency();
    lastSpawnTime = frameCount;
  }
  
  drawLanes();

  for (let i = notes.length - 1; i >= 0; i--) {
    let n = notes[i];
    n.update();
    n.show();

    if (n.y > height + 50) {
      notes.splice(i, 1);
    }
  }

  for (let i = particles.length - 1; i >= 0; i--) {
    let p = particles[i];
    p.update();
    p.show();
    if (p.isDead()) {
      particles.splice(i, 1);
    }
  }

  stroke(255, 200, 0);
  strokeWeight(3);
  line(0, hitLineY, width, hitLineY);

  noStroke();
  fill(255);
  text("D", laneWidth * 0.5, hitLineY + 20);
  text("F", laneWidth * 1.5, hitLineY + 20);
  text("J", laneWidth * 2.5, hitLineY + 20);
  text("K", laneWidth * 3.5, hitLineY + 20);
}

function drawLanes() {
  stroke(80);
  strokeWeight(2);
  for (let i = 0; i < lanes; i++) {
    let x = i * laneWidth;
    line(x, 0, x, height);
  }
}

function spawnNoteByFrequency() {
  let bass = fft.getEnergy("bass");
  let lowMid = fft.getEnergy("lowMid");
  let mid = fft.getEnergy("mid");
  let treble = fft.getEnergy("treble");

  let maxEnergy = Math.max(bass, lowMid, mid, treble);
  let laneIndex = 0;

  if (maxEnergy === bass) laneIndex = 0;
  else if (maxEnergy === lowMid) laneIndex = 1;
  else if (maxEnergy === mid) laneIndex = 2;
  else laneIndex = 3;

  let x = laneIndex * laneWidth + laneWidth / 2;
  notes.push(new Note(x, -20, laneIndex));
}

function createExplosion(x, y) {
  for (let i = 0; i < 15; i++) {
    particles.push(new Particle(x, y));
  }
}

class Note {
  constructor(x, y, laneIndex) {
    this.x = x;
    this.y = y;
    this.laneIndex = laneIndex;
    this.speed = noteSpeed;
    this.hit = false;
  }

  update() {
    this.y += this.speed;
  }

  show() {
    if (this.hit) return;

    noStroke();
    if (this.laneIndex === 0) fill(255, 100, 100); 
    else if (this.laneIndex === 1) fill(100, 255, 100);
    else if (this.laneIndex === 2) fill(100, 100, 255);
    else fill(255, 255, 100);

    ellipse(this.x, this.y, laneWidth * 0.6, laneWidth * 0.6);
  }
}

class Particle {
  constructor(x, y) {
    this.x = x;
    this.y = y;
    this.vx = random(-3, 3);
    this.vy = random(-3, 3);
    this.alpha = 255;
    this.color = color(255, random(200, 255), 0);
    this.size = random(5, 15);
  }

  update() {
    this.x += this.vx;
    this.y += this.vy;
    this.alpha -= 8;
    this.size *= 0.92;
  }

  show() {
    noStroke();
    fill(red(this.color), green(this.color), blue(this.color), this.alpha);
    ellipse(this.x, this.y, this.size);
  }

  isDead() {
    return this.alpha <= 0;
  }
}

function mousePressed() {
  if (song.isLoaded() && !song.isPlaying()) {
    song.play();
  }
}

function keyPressed() {
  if (key === "s") saveCanvas("myGame", "png");
  let keyUpper = key.toUpperCase();
  if (keyUpper in keyToLane) {
    handleHit(keyToLane[keyUpper]);
  }
}

function handleHit(laneIndex) {
  let closestNote = null;
  let closestDistance = Infinity;

  for (let n of notes) {
    if (n.laneIndex === laneIndex && !n.hit) {
      let d = abs(n.y - hitLineY);
      if (d < closestDistance) {
        closestDistance = d;
        closestNote = n;
      }
    }
  }

  if (closestNote && closestDistance <= hitWindow) {
    closestNote.hit = true;
    createExplosion(closestNote.x, closestNote.y);
    playLaneSound(laneIndex);
  }
}

function playLaneSound(laneIndex) {
  let osc = oscillators[laneIndex];
  osc.amp(0.4, 0.01);
  osc.amp(0, 0.2);
}

Generate notes based on audio accents and pitch

I mainly set up two things: fft and peakDetect, which are used to analyze the spectrum and rhythm respectively. Then, each time an accent is reached, notes are generated on four tracks based on the range of pitch.

function draw() {
  background(30); 

 
  if (!song.isPlaying()) {
    fill(255);
    text("Click to Start Music", width/2, height/2);
    return;
  }
  
  fft.analyze();           
  peakDetect.update(fft);
    
  if (peakDetect.isDetected && frameCount > lastSpawnTime + spawnCooldown) {
    spawnNoteByFrequency(); 
    lastSpawnTime = frameCount;
  }
  }

function spawnNoteByFrequency() {
  let bass = fft.getEnergy("bass");
  let lowMid = fft.getEnergy("lowMid");
  let mid = fft.getEnergy("mid");
  let treble = fft.getEnergy("treble");

  let maxEnergy = Math.max(bass, lowMid, mid, treble);
  let laneIndex = 0;

  if (maxEnergy === bass) laneIndex = 0;
  else if (maxEnergy === lowMid) laneIndex = 1;
  else if (maxEnergy === mid) laneIndex = 2;
  else laneIndex = 3;

  let x = laneIndex * laneWidth + laneWidth / 2;
  notes.push(new Note(x, -20, laneIndex));
}

Particle effects when notes hit

I bind each particle to a class, and then update it every frame in the update function.

class Particle {
  constructor(x, y) {
    this.x = x;
    this.y = y;
    this.vx = random(-3, 3);
    this.vy = random(-3, 3);
    this.alpha = 255;
    this.color = color(255, random(200, 255), 0);
    this.size = random(5, 15);
  }

  update() {
    this.x += this.vx;
    this.y += this.vy;
    this.alpha -= 8;
    this.size *= 0.92;
  }

  show() {
    noStroke();
    fill(red(this.color), green(this.color), blue(this.color), this.alpha);
    ellipse(this.x, this.y, this.size);
  }

  isDead() {
    return this.alpha <= 0;
  }
}

function createExplosion(x, y) {
  for (let i = 0; i < 15; i++) {
    particles.push(new Particle(x, y));
  }
}

function handleHit(laneIndex) {
  let closestNote = null;
  let closestDistance = Infinity;

  for (let n of notes) {
    if (n.laneIndex === laneIndex && !n.hit) {
      let d = abs(n.y - hitLineY);
      if (d < closestDistance) {
        closestDistance = d;
        closestNote = n;
      }
    }
  }

  if (closestNote && closestDistance <= hitWindow) {
    closestNote.hit = true;
    createExplosion(closestNote.x, closestNote.y);
    playLaneSound(laneIndex);
  }
}

11/24 Update

Today, while running the program, I discovered that the notes only appeared on the first two tracks. I tried to find out why.The problem wasn't with my code; it was with the logic. Most music and songs are concentrated in the low to mid frequencies, so analyzing music using energy would inevitably result in only the first two tracks generating notes. Then, I changed the analysis method from energy to pitch, and that stopped the process. The main reason for this is that I'm musically illiterate…

function spawnNoteByFrequency() {
  let bass = fft.getEnergy("bass");
  let lowMid = fft.getEnergy("lowMid");
  let mid = fft.getEnergy("mid");
  let treble = fft.getEnergy("treble");

  let maxEnergy = Math.max(bass, lowMid, mid, treble);
  let laneIndex = 0;

  if (maxEnergy === bass) laneIndex = 0;
  else if (maxEnergy === lowMid) laneIndex = 1;
  else if (maxEnergy === mid) laneIndex = 2;
  else laneIndex = 3;

  let x = laneIndex * laneWidth + laneWidth / 2;
  notes.push(new Note(x, -20, laneIndex));
}

This is the previous code.

function spawnNoteByFrequency() {
  if (!spectrumData || spectrumData.length === 0) return;

  let nyquist = sampleRate() / 2;
  let binWidth = nyquist / spectrumData.length;

  let maxAmp = 0;
  let maxIndex = 0;

  for (let i = 0; i < spectrumData.length; i++) {
    if (spectrumData[i] > maxAmp) {
      maxAmp = spectrumData[i];
      maxIndex = i;
    }
  }

  let dominantFreq = maxIndex * binWidth;

  let laneIndex = 0;
  if (dominantFreq > 0) {
    let midi = 69 + 12 * (Math.log(dominantFreq / 440) / Math.log(2));
    let m = Math.round(midi);
    laneIndex = ((m % lanes) + lanes) % lanes;
  }

  let x = laneIndex * laneWidth + laneWidth / 2;
  notes.push(new Note(x, -20, laneIndex));
}

new function