Handsfree.js

# Plugin: palmPointers

  • 🖐 With your palm(s) pointed at the screen, move your hands to move the pointer
  • 👌 Pinch your index and thumb to scroll the area under the pointer
  • Try scrolling two scroll areas at once!

Models: MediaPipe Hands

About: This plugin displays a pointer on the screen for each visible hand, which can be used as a guide when pinch clicking or pinch scrolling

Activate: handsfree.plugin.palmPointers.enable() or handsfree.enablePlugins('core')

Tags: ['core']

# Try scrolling these different areas at the same time

# The Pointers

This plugin adds handsfree.data.hands.pointers to the Hands Model with an object for each hand:

handsfree.data.hands.pointers = [
  // Left hand 1
  {x, y, isVisible},
  // Right hand 1
  {x, y, isVisible},
  // Left hand 2
  {x, y, isVisible},
  // Right hand 2
  {x, y, isVisible}
]

The pointers are automatically shown and hidden as the hands come in and out view. You can access these in several ways:

// From anywhere
handsfree.data.hands.pointers[0]

// From inside a plugin
handsfree.use('logger', data => {
  if (!data.hands) return

  console.log(data.hands.pointers[0])
})

// From an event
document.addEventListener('handsfree-data', event => {
  const data = event.detail
  if (!data.hands) return

  console.log(data.hands.pointers[0])
})

# Config

# During instantiation

handsfree = new Handsfree({
  hands: true,

  plugin: {
    palmPointers: {
      enabled: true,

      // How much to offfset the pointers by
      // - This is useful for when the camera won't be in front of you
      // - This is also useful when working with multiple displays
      offset: {
        x: 0,
        y: 0
      },

      // A multiplier to apply to moving the pointer
      speed: {
        x: 1.5,
        y: 1.5
      }
    }
  }
})

# After instantiation

handsfree = new Handsfree({hands: true})
handsfree.start()

handsfree.plugin.palmPointers.enable()
handsfree.plugin.palmPointers.speed = {x: 2, y: 2}
handsfree.plugin.palmPointers.offset = {x: 100, y: 100}

# Properties

The following properties are available on this plugin:

// The pointer {x, y} for each hand
handsfree.plugin.palmPointers.pointers === [leftHand1, rightHand1, leftHand2, rightHand2]

// The pointer elements
handsfree.plugin.palmPointers.$pointers === [leftHand1, rightHand1, leftHand2, rightHand2]

# Examples of accessing properties

// hide the pointer for the left hand (for the 1st person)
handsfree.plugin.palmPointers.$pointers[0].style.display = 'none'

// show the pointer for the right hand (for the 2nd person)
handsfree.plugin.palmPointers.$pointers[3].style.display = 'block'

# Methods

The following methods are available on this plugins:

// Show the pointers (shown by default when enabled)
this.plugin.palmPointers.showPointers()

// Hide pointers. Note that the pointer values are still updated, this simply hides
// them visually
this.plugin.palmPointers.hidePointers()

# See also

  • pinchers - A collection of events, properties, and helper styles for finger pinching
  • pinchScroll - Scroll the page by pinching together your thumb and pointer finger

# Full plugin code

// Maps handsfree pincher events to 
const eventMap = {
  start: 'mousedown',
  held: 'mousemove',
  released: 'mouseup'
}

// The last pointer positions for each hand, used to determine movement over time
let lastHeld = [{x: 0, y: 0}, {x: 0, y: 0}, {x: 0, y: 0}, {x: 0, y: 0}]

/**
 * Move a pointer with your palm
 */
export default {
  models: 'hands',
  tags: ['browser'],
  enabled: false,

  // The pointer element
  $pointer: [],
  arePointersVisible: true,

  // Pointers position
  pointer: [
    { x: -20, y: -20, isVisible: false },
    { x: -20, y: -20, isVisible: false },
    { x: -20, y: -20, isVisible: false },
    { x: -20, y: -20, isVisible: false }
  ],

  // Used to smoothen out the pointer
  tween: [
    {x: -20, y: -20},
    {x: -20, y: -20},
    {x: -20, y: -20},
    {x: -20, y: -20},
  ],

  config: {
    offset: {
      x: 0,
      y: 0
    },

    speed: {
      x: 1,
      y: 1
    }
  },

  /**
   * Create and toggle pointers
   */
  onUse () {
    for (let i = 0; i < 4; i++) {
      const $pointer = document.createElement('div')
      $pointer.classList.add('handsfree-pointer', 'handsfree-pointer-palm', 'handsfree-hide-when-started-without-hands')
      document.body.appendChild($pointer)
      this.$pointer[i] = $pointer
    }
    
    if (this.enabled && this.arePointersVisible) {
      this.showPointers()
    } else {
      this.hidePointers()
    }
  },

  /**
   * Show pointers on enable
   */
  onEnable () {
    const arePointersVisible = this.arePointersVisible
    this.showPointers()
    this.arePointersVisible = arePointersVisible
  },

  /**
   * Hide pointers on disable
   */
  onDisable () {
    const arePointersVisible = this.arePointersVisible
    this.hidePointers()
    this.arePointersVisible = arePointersVisible
  },

  /**
   * Positions the pointer and dispatches events
   */
  onFrame ({hands}) {
    // Hide pointers
    if (!hands?.multiHandLandmarks) {
      this.$pointer.forEach($pointer => $pointer.style.display = 'none')
      return
    }

    hands.pointer = [
      { isVisible: false },
      { isVisible: false },
      { isVisible: false },
      { isVisible: false }
    ]
    
    hands.multiHandLandmarks.forEach((landmarks, n) => {
      const pointer = hands.pointer[n]

      // Use the correct hand index
      let hand
      if (n < 2) {
        hand = hands.multiHandedness[n].label === 'Right' ? 0 : 1
      } else {
        hand = hands.multiHandedness[n].label === 'Right' ? 2 : 3
      }

      // Update pointer position
      this.handsfree.TweenMax.to(this.tween[hand], 1, {
        x: window.outerWidth * this.config.speed.x
          - window.outerWidth * this.config.speed.x / 2
          + window.outerWidth / 2
          - hands.multiHandLandmarks[n][21].x * this.config.speed.x * window.outerWidth
          + this.config.offset.x,
        y: hands.multiHandLandmarks[n][21].y * window.outerHeight * this.config.speed.y
          - window.outerHeight * this.config.speed.y / 2
          + window.outerHeight / 2
          + this.config.offset.y,
        overwrite: true,
        ease: 'linear.easeNone',
        immediate: true
      })

      hands.pointer[hand] = {
        x: this.tween[hand].x,
        y: this.tween[hand].y,
        isVisible: true
      }

      // Visually update pointer element
      this.$pointer[hand].style.left = `${this.tween[hand].x}px`
      this.$pointer[hand].style.top = `${this.tween[hand].y}px`

      // Dispatch events
      let event = pointer?.pinchState?.[n]?.[0]
      if (event && pointer.isVisible) {
        // Get the event and element to send events to
        event = eventMap[event]
        const $el = document.elementFromPoint(pointer.x, pointer.y)
        
        // Dispatch the event
        if ($el) {
          $el.dispatchEvent(
            new MouseEvent(event, {
              view: window,
              button: 0,
              bubbles: true,
              cancelable: true,
              clientX: pointer.x,
              clientY: pointer.y,
              // Only used when the mouse is captured in full screen mode
              movementX: pointer.x - lastHeld[hand].x,
              movementY: pointer.y - lastHeld[hand].y
            })
          )
        }

        lastHeld[hand] = pointer
      }
    })

    // Toggle pointers
    hands.pointer.forEach((pointer, hand) => {
      if (pointer.isVisible) {
        this.$pointer[hand].style.display = 'block'
      } else {
        this.$pointer[hand].style.display = 'none'
      }
    })
  },

  /**
   * Toggle pointer
   */
  onDisable() {
    this.$pointer.forEach($pointer => {
      $pointer.classList.add('handsfree-hidden')
    })
  },

  /**
   * Toggle pointers
   */
  showPointers () {
    this.arePointersVisible = true
    for (let i = 0; i < 4; i++) {
      this.$pointer[i].classList.remove('handsfree-hidden')
    }
  },
  hidePointers () {
    this.arePointersVisible = false
    for (let i = 0; i < 4; i++) {
      this.$pointer[i].classList.add('handsfree-hidden')
    }
  }
}

← pinchers →

Debugger