Figured it out. It's a bug with the AppleScript (maybe that was obvious?), so I changed from using the Cocoa framework to the Foundation and AppKit frameworks. The following AppleScript code works, but not for the fn key. Since I pretty much only care about the option key, it doesn't matter at all for me.
use framework "Foundation"
use framework "AppKit"
property NSCapsLockMask : a reference to 65536
property NSShiftKeyMask : a reference to 131072
property NSControlKeyMask : a reference to 262144
property NSOptionKeyMask : a reference to 524288
property NSCommandKeyMask : a reference to 1048576
property NSFunctionKeyMask : a reference to 8388608
property NSEvent : a reference to current application's NSEvent
on KeyPressed(KeyName)
if KeyName = "caps lock" then
set TheMask to NSCapsLockMask
else if KeyName = "shift" then
set TheMask to NSShiftKeyMask
else if KeyName = "control" then
set TheMask to NSControlKeyMask
else if KeyName = "option" then
set TheMask to NSOptionKeyMask
else if KeyName = "command" then
set TheMask to NSCommandKeyMask
else
return false
end if
set MFlags to NSEvent's modifierFlags() as integer
if ((MFlags div TheMask) mod 2) = 0 then
return false
else
return true
end if
end KeyPressed
on getModifiersDown()
set modifierKeysDown to {command_down:false, option_down:false, control_down:false, shift_down:false, capslock_down:false}
set KeyFlags to NSEvent's modifierFlags() as integer
if ((KeyFlags div NSCommandKeyMask) mod 2) is not 0 then
set command_down of modifierKeysDown to true
end if
if ((KeyFlags div NSOptionKeyMask) mod 2) is not 0 then
set option_down of modifierKeysDown to true
end if
if ((KeyFlags div NSControlKeyMask) mod 2) is not 0 then
set control_down of modifierKeysDown to true
end if
if ((KeyFlags div NSShiftKeyMask) mod 2) is not 0 then
set shift_down of modifierKeysDown to true
end if
if ((KeyFlags div NSCapsLockMask) mod 2) is not 0 then
set capslock_down of modifierKeysDown to true
end if
# NOTE: Buggy
# Function key erroneously sets to true when the last key pressed was an arrow key
#if ((KeyFlags div NSFunctionKeyMask) mod 2) is not 0 then
# set fn_down of modifierKeysDown to true
#end if
return modifierKeysDown
end getModifiersDown
set modifierKeysDown to getModifiersDown()
if (option_down of modifierKeysDown is false) then
tell application "BetterTouchTool" to trigger_action "{\"BTTPredefinedActionType\":24}"
else
tell application "BetterTouchTool" to trigger_action "{\"BTTPredefinedActionType\":198}"
end if
return modifierKeysDown